$ cd lodcc/
$ pip install -r requirements.txt
$ sudo apt-get install dtrx raptor2-utils
- Small dataset
python lodcc.py --parse-resource-urls --use-datasets museums-in-italy --log-level-debug
- Larger dataset
python lodcc.py --parse-resource-urls --use-datasets pokepedia-fr --log-level-debug
- Multithreaded processing
python lodcc.py --parse-resource-urls --use-datasets museums-in-italy pokepedia-fr --threads 2
- Determine file sizes of all dumps
$ find dumps/ -type f -exec ls -s --block-size=M {} \; > dumps-sizes.txt
$ cat dumps-sizes.txt | sed -e '/edgelist/! s/^.*$/###/' -e '/^###/D' | sort -h -r | less
- Download all filenames of all datasets into a file
dbpedia-link.txt
$ curl -L http://downloads.dbpedia.org/2016-10/core-i18n/en/ -o dbpedia-link.txt
- Filter unnecessary datasets and prefix with url
$ cat dbpedia-link.txt | cut -d '"' -f2 | egrep -i "ttl" | egrep -i -v "wkd|sorted|nested" | sed 's#^\(.*\)#http://downloads.dbpedia.org/2016-10/core-i18n/en/\1#' | sed -n '2,60p' > dbpedia-links.txt