#!/bin/bash



input_dir="dbpedia-generic-snapshot-2022-12-files"
output_file="dbpedia-generic-snapshot-2022-12.ttl.zst"
log_file="dbpedia-generic-snapshot-2022-12.ttl.zst-files.txt"

num_cores=$(nproc --all)
zstd_cores=$((num_cores / 2))

# Clear the log file if it exists
> "$log_file"

# Find all relevant files and process them
find "$input_dir" -type f -print0 | while IFS= read -r -d '' file; do
  # Get relative path, size, and hash of each file
  relative_path=$(realpath --relative-to="$input_dir" "$file")
  file_size=$(stat -c%s "$file")
  file_hash=$(sha256sum "$file" | awk '{print $1}')

  # Log the information to input_files.txt
  echo "$relative_path, $file_size bytes, $file_hash" >> "$log_file"

  # Process the files based on their extension
  case "$file" in
    *.ttl.bz2|*.nt.bz2|*.ttl.bzip2|*.nt.bzip2)
      pbzip2 -m5000 -p${zstd_cores} -dc "$file"
      ;;
    *.ttl.gz|*.nt.gz)
      pigz -p ${zstd_cores} -dc "$file"
      ;;
    *.nt|*.ttl)
      cat "$file"
      ;;
  esac
done | zstd -T0${zstd_cores} --ultra -22 -c -o "$output_file"

