#!/bin/bash
set -eu
# Define the URL and file pattern
BASE_URL="https://downloads.dbpedia.org/2016-10/core-i18n/en/"
FILE_PATTERN="*.ttl.bz2"

# Create a directory to store downloaded files
DOWNLOAD_DIR="dbpedia_downloads"
mkdir -p "$DOWNLOAD_DIR"
cd "$DOWNLOAD_DIR" || exit

# Get the list of .ttl.bz2 files from the URL
echo "Fetching list of files..."
wget -q -O- "$BASE_URL" | grep -oP 'href="\K[^"]*\.ttl\.bz2' > files.txt

# Download the files using wget with parallel
echo "Starting download of .ttl.bz2 files..."
cat /upb/users/q/quannian/profiles/unix/cs/dbdump/dbpedia_downloads/files.txt | parallel -j 0 wget -q --show-progress "$BASE_URL{}"

# Extract the downloaded .ttl.bz2 files using lbzip2 with parallel
echo "Extracting .ttl files from .ttl.bz2 archives..."
ls *.ttl.bz2 | parallel -j 0 lbzip2 -dk --verbose

# Remove the .ttl.bz2 files after extraction
echo "Cleaning up .ttl.bz2 files..."
rm *.ttl.bz2

echo "Download and extraction completed."
