👷 (ingest-file) Build from base docker image

.github/workflows/ingest-file.yml

@@ -25,7 +25,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ghcr.io/investigativedata/ingest-file
+          images: ghcr.io/investigativedata/aleph-ingest-file
           tags: |
             type=ref,event=branch
             type=semver,pattern={{version}}

aleph-ingest/Dockerfile

@@ -1,151 +1,13 @@
-FROM ubuntu:22.04
+FROM ghcr.io/investigativedata/ingest-file-spacy
 ENV DEBIAN_FRONTEND noninteractive
 
 LABEL org.opencontainers.image.title "FollowTheMoney File Ingestors"
 LABEL org.opencontainers.image.licenses MIT
-LABEL org.opencontainers.image.source https://github.com/alephdata/ingest-file
+LABEL org.opencontainers.image.source https://github.com/investigativedata/aleph-ingest-file
 
-# Enable non-free archive for `unrar`.
-# RUN echo "deb http://http.us.debian.org/debian bookworm non-free" >/etc/apt/sources.list.d/nonfree.list
-RUN apt-get -qq -y update \
-    && apt-get -qq -y install build-essential locales ca-certificates \
-    # python deps (mostly to install their dependencies)
-    python3-pip python3-dev python3-pil \
-    # tesseract
-    tesseract-ocr libtesseract-dev libleptonica-dev pkg-config\
-    # libraries
-    libxslt1-dev libpq-dev libldap2-dev libsasl2-dev \
-    zlib1g-dev libicu-dev libxml2-dev \
-    # package tools
-    unrar p7zip-full \
-    # audio & video metadata
-    libmediainfo-dev \
-    # image processing, djvu
-    imagemagick-common imagemagick mdbtools djvulibre-bin \
-    libtiff5-dev libjpeg-dev libfreetype6-dev libwebp-dev \
-    libtiff-tools ghostscript librsvg2-bin jbig2dec \
-    pst-utils \
-    ### tesseract 
-    tesseract-ocr-eng \
-    tesseract-ocr-swa \
-    tesseract-ocr-swe \
-    # tesseract-ocr-tam \
-    # tesseract-ocr-tel \
-    tesseract-ocr-fil \
-    # tesseract-ocr-tha \
-    tesseract-ocr-tur \
-    tesseract-ocr-ukr \
-    # tesseract-ocr-vie \
-    tesseract-ocr-nld \
-    tesseract-ocr-nor \
-    tesseract-ocr-pol \
-    tesseract-ocr-por \
-    tesseract-ocr-ron \
-    tesseract-ocr-rus \
-    tesseract-ocr-slk \
-    tesseract-ocr-slv \
-    tesseract-ocr-spa \
-    # tesseract-ocr-spa_old \
-    tesseract-ocr-sqi \
-    tesseract-ocr-srp \
-    tesseract-ocr-ind \
-    tesseract-ocr-isl \
-    tesseract-ocr-ita \
-    # tesseract-ocr-ita_old \
-    # tesseract-ocr-jpn \
-    tesseract-ocr-kan \
-    tesseract-ocr-kat \
-    # tesseract-ocr-kor \
-    tesseract-ocr-khm  \
-    tesseract-ocr-lav \
-    tesseract-ocr-lit \
-    # tesseract-ocr-mal \
-    tesseract-ocr-mkd \
-    tesseract-ocr-mya \
-    tesseract-ocr-mlt \
-    tesseract-ocr-msa \
-    tesseract-ocr-est \
-    # tesseract-ocr-eus \
-    tesseract-ocr-fin \
-    tesseract-ocr-fra \
-    tesseract-ocr-frk \
-    # tesseract-ocr-frm \
-    # tesseract-ocr-glg \
-    # tesseract-ocr-grc \
-    tesseract-ocr-heb \
-    tesseract-ocr-hin \
-    tesseract-ocr-hrv \
-    tesseract-ocr-hye \
-    tesseract-ocr-hun \
-    # tesseract-ocr-ben \
-    tesseract-ocr-bul \
-    tesseract-ocr-cat \
-    tesseract-ocr-ces \
-    tesseract-ocr-nep \
-    # tesseract-ocr-chi_sim \
-    # tesseract-ocr-chi_tra \
-    # tesseract-ocr-chr \
-    tesseract-ocr-dan \
-    tesseract-ocr-deu \
-    tesseract-ocr-ell \
-    # tesseract-ocr-enm \
-    # tesseract-ocr-epo \
-    # tesseract-ocr-equ \
-    tesseract-ocr-afr \
-    tesseract-ocr-ara \
-    tesseract-ocr-aze \
-    tesseract-ocr-bel \
-    tesseract-ocr-uzb \
-    ### pdf convert: libreoffice + a bunch of fonts 
-    libreoffice fonts-opensymbol hyphen-fr hyphen-de \
-    hyphen-en-us hyphen-it hyphen-ru fonts-dejavu fonts-dejavu-core fonts-dejavu-extra \
-    fonts-droid-fallback fonts-dustin fonts-f500 fonts-fanwood fonts-freefont-ttf \
-    fonts-liberation fonts-lmodern fonts-lyx fonts-sil-gentium fonts-texgyre \
-    fonts-tlwg-purisa \
-    ###
-    && apt-get -qq -y autoremove \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
-
-# Set up the locale and make sure the system uses unicode for the file system.
-ENV LANG='en_US.UTF-8' \
-    TZ='UTC' \
-    OMP_THREAD_LIMIT='1' \
-    OPENBLAS_NUM_THREADS='1'
-
-RUN groupadd -g 1000 -r app \
-    && useradd -m -u 1000 -s /bin/false -g app app
-
-# Download the ftm-typepredict model
-RUN mkdir /models/ && \
-    curl -o "/models/model_type_prediction.ftz" "https://public.data.occrp.org/develop/models/types/type-08012020-7a69d1b.ftz"
-
-# Having updated pip/setuptools seems to break the test run for some reason (12/01/2022)
-# RUN pip3 install --no-cache-dir -U pip setuptools
-COPY ./ingest-file/requirements.txt /tmp/
-RUN pip3 install --no-cache-dir --prefer-binary --upgrade pip
-RUN pip3 install --no-cache-dir --prefer-binary --upgrade setuptools wheel
+COPY ./requirements.txt /tmp/
 RUN pip3 install --no-cache-dir --prefer-binary -r /tmp/requirements.txt
 
-# Install spaCy models
-RUN python3 -m spacy download en_core_web_sm \
-    && python3 -m spacy download de_core_news_sm \
-    && python3 -m spacy download fr_core_news_sm \
-    && python3 -m spacy download es_core_news_sm
-RUN python3 -m spacy download ru_core_news_sm \
-    && python3 -m spacy download pt_core_news_sm \
-    && python3 -m spacy download ro_core_news_sm \
-    && python3 -m spacy download mk_core_news_sm 
-RUN python3 -m spacy download el_core_news_sm \
-    && python3 -m spacy download pl_core_news_sm \
-    && python3 -m spacy download it_core_news_sm \
-    && python3 -m spacy download lt_core_news_sm \
-    && python3 -m spacy download nl_core_news_sm \
-    && python3 -m spacy download nb_core_news_sm \
-    && python3 -m spacy download da_core_news_sm
-# RUN python3 -m spacy download zh_core_web_sm
-
 COPY ./ingest-file /ingestors
 WORKDIR /ingestors
 RUN pip3 install --no-cache-dir --config-settings editable_mode=compat --use-pep517 -e /ingestors

aleph-ingest/requirements.txt

@@ -0,0 +1,41 @@
+banal==1.0.6
+normality==2.5.0
+pantomime==0.6.1
+followthemoney==3.5.8
+followthemoney-store[postgresql]==3.0.6
+servicelayer[google,amazon]==1.22.1
+languagecodes==1.1.1
+countrytagger==0.1.2
+pyicu==2.12
+google-cloud-vision==3.5.0
+tesserocr==2.6.2
+spacy==3.6.1
+fingerprints==1.1.1
+fasttext==0.9.2
+
+# Development
+pytest==7.4.4
+pytest-cov==4.1.0
+click==8.1.6
+
+# File format support
+dbf==0.99.9
+pymediainfo==6.1.0
+python-magic==0.4.27
+rarfile==4.1
+xlrd==2.0.1
+openpyxl==3.1.2
+odfpy==1.4.1
+faust-cchardet
+lxml==5.0.0
+olefile==0.47
+Pillow==10.1.0
+vobject==0.9.6.1
+msglite==0.30.0
+icalendar==5.0.11
+
+cryptography==41.0.7
+requests[security]==2.31.0
+pymupdf==1.21.1
+
+sentry_sdk==1.39.1