Update CIALUG_December2024.md

CIALUG_December2024.md

@@ -310,3 +310,224 @@ int main() {
     }
 }
 ```
+
+
+
+* additions for named pipes and GNU parallel
+
+
+## Advanced IPC and Parallel Processing Examples
+
+### Named Pipes (FIFOs) Deep Dive
+
+```bash
+#!/bin/bash
+# Demonstrating different named pipe patterns
+
+# 1. Basic producer-consumer with backpressure
+setup_basic_pipe() {
+    mkfifo /tmp/dataflow
+    # Cleanup on exit
+    trap "rm -f /tmp/dataflow" EXIT
+}
+
+# Producer with rate limiting
+producer() {
+    local rate=$1  # Messages per second
+    local sleep_time=$(bc <<< "scale=4; 1/$rate")
+
+    for i in {1..100}; do
+        echo "Message $i"
+        sleep "$sleep_time"
+    done > /tmp/dataflow
+}
+
+# Consumer with processing time
+consumer() {
+    local process_time=$1
+    while read line; do
+        echo "[$(date +%T.%N)] Received: $line"
+        sleep "$process_time"
+    done < /tmp/dataflow
+}
+
+# 2. Multiple consumers (round-robin)
+setup_multi_consumer() {
+    mkfifo /tmp/pipe1 /tmp/pipe2
+    trap "rm -f /tmp/pipe1 /tmp/pipe2" EXIT
+}
+
+multi_producer() {
+    for i in {1..100}; do
+        echo "Data $i" > /tmp/pipe1
+        echo "Data $i" > /tmp/pipe2
+    done
+}
+
+consumer_with_id() {
+    local id=$1
+    local pipe="/tmp/pipe$id"
+    while read line; do
+        echo "Consumer $id: $line"
+        sleep 0.1  # Simulate processing
+    done < "$pipe"
+}
+
+# 3. Bidirectional communication
+setup_bidirectional() {
+    mkfifo /tmp/request /tmp/response
+    trap "rm -f /tmp/request /tmp/response" EXIT
+}
+
+server() {
+    while true; do
+        read request < /tmp/request
+        echo "Processing: $request"
+        echo "Response to: $request" > /tmp/response
+    done
+}
+
+client() {
+    for i in {1..5}; do
+        echo "Request $i" > /tmp/request
+        read response < /tmp/response
+        echo "Got: $response"
+    done
+}
+
+# Example usage:
+# setup_basic_pipe
+# producer 2 & consumer 0.1
+```
+
+### GNU Parallel Advanced Patterns
+
+```bash
+#!/bin/bash
+
+# 1. Processing files with custom output handling
+process_large_file() {
+    local input=$1
+    local chunk_size=1000  # lines per chunk
+
+    # Split input into chunks while maintaining line integrity
+    split -l "$chunk_size" "$input" /tmp/chunk_
+
+    # Process chunks in parallel
+    ls /tmp/chunk_* | parallel -j$(nproc) --progress \
+        'cat {} | while read line; do
+            echo "Processing $line" >&2
+            echo "$line" | md5sum
+        done > {}.processed'
+
+    # Combine results
+    cat /tmp/chunk_*.processed > "${input}.processed"
+    rm /tmp/chunk_* /tmp/chunk_*.processed
+}
+
+# 2. Parallel data transformation with SQL-like operations
+parallel_transform() {
+    local input=$1
+
+    # Generate work units
+    seq 1 100 | \
+    parallel -j$(nproc) --pipe --block 1M \
+        "awk '{
+            sum += \$1;
+            count++;
+        }
+        END {
+            if (count > 0) 
+                print sum/count
+        }'" > results.txt
+}
+
+# 3. Network bandwidth testing with parallel connections
+parallel_network_test() {
+    local target=$1
+    local num_connections=${2:-10}
+
+    seq "$num_connections" | \
+    parallel -j"$num_connections" \
+        "curl -s -w '%{speed_download}\n' -o /dev/null $target"
+}
+
+# 4. Directory tree processing with custom control
+parallel_tree_process() {
+    local dir=$1
+
+    # Find all files and process based on type
+    find "$dir" -type f -print0 | \
+    parallel -0 -j$(nproc) --bar \
+        'file={};
+         ext="${file##*.}";
+         case "$ext" in
+            "jpg"|"png")
+                convert "$file" -resize "50%" "${file%.}.thumb.$ext"
+                ;;
+            "txt"|"log")
+                gzip -9 "$file"
+                ;;
+            *)
+                echo "Skipping $file" >&2
+                ;;
+         esac'
+}
+
+# 5. Memory-aware parallel processing
+parallel_memory_aware() {
+    local input_dir=$1
+    local mem_per_job="1G"  # Memory per job
+
+    # Calculate jobs based on available memory
+    local total_mem=$(free -g | awk '/^Mem:/{print $2}')
+    local max_jobs=$((total_mem * 1024 * 1024 * 1024 / (1024 * 1024 * 1024)))  # Convert GB to jobs
+
+    find "$input_dir" -type f | \
+    parallel --memfree "$mem_per_job" \
+             --jobs "$max_jobs" \
+             --progress \
+        'process_file {}'
+}
+
+# Example usage functions
+demonstrate_pipes() {
+    echo "Setting up named pipes..."
+    setup_basic_pipe
+    producer 2 & consumer 0.1
+    wait
+}
+
+demonstrate_parallel() {
+    echo "Running parallel processing examples..."
+    # Create test data
+    seq 1000 > testdata.txt
+    parallel_transform testdata.txt
+    rm testdata.txt results.txt
+}
+
+# Run demonstrations
+# demonstrate_pipes
+# demonstrate_parallel
+```
+
+This code adds:
+
+1. Advanced named pipe patterns:
+   - Basic producer-consumer with rate limiting
+   - Multiple consumer setup
+   - Bidirectional communication
+
+2. GNU Parallel patterns:
+   - Custom file chunk processing
+   - Memory-aware parallel execution
+   - Network testing
+   - Directory tree processing
+   - Data transformation pipelines
+
+Would you like me to:
+1. Add more specific use cases for any of these patterns?
+2. Include performance measurement code?
+3. Add error handling and recovery examples?
+4. Include more complex data processing patterns?
+