sites.yml

sites:
  - name: occrp
    urls:
      - https://occrp.org
      - https://www.occrp.org/en/investigations
      - https://www.occrp.org/en/daily
      - https://www.occrp.org/en/blog
      - https://www.occrp.org/en/projects/
    crawl:
      and:
        - or:
            - domain: reportingproject.net
            - domain: occrp.org
        - not:
            or:
              - mime: nonweb
              - domain: vis.occrp.org
              - domain: aleph.occrp.org
              - domain: admin.occrp.org
              - domain: id.occrp.org
              - domain: cdn.occrp.org
              - prefix: https://www.occrp.org/images
              - prefix: https://www.occrp.org/documents/
    parse:
      or:
        - element: .//article//li[@class="authors"]
        - element: .//aside[@class="byline"]
        - element: .//section[@class="blog"]
        - element: .//div[@class="occrp-story"]

  - name: icij
    urls:
      - https://www.icij.org/
      - https://www.icij.org/inside-icij/
    domain_concurrency: 1
    delay: 1
    query_ignore:
      - utm_term
      - utm_source
      - utm_campaign
      - utm_medium
    crawl:
      and:
        - or:
            - domain: icij.org
        - not:
            or:
              - mime: nonweb
              - domain: offshoreleaks.icij.org
              - domain: medicaldevices.icij.org
              - domain: wwwmaster.icij.org
              - domain: media.icij.org
              - domain: dev.icij.org
              - domain: datashare.icij.org
              - domain: donate.icij.org
              - domain: cloudfront-4.icij.org
              - domain: wwww.icij.org
    parse:
      or:
        - element: .//time[@class="post-published"]

  - name: dossier_center
    urls:
      - https://dossier.center/
      - https://dossier.center/investigations/
      - https://dossier.center/persons/
      - https://fsb.dossier.center/
      - https://fsb.dossier.center/content/
    crawl:
      and:
        - or:
            - domain: dossier.center
        - not:
          or:
            - mime: nonweb
    parse:
      and:
        - or:
            - element: .//meta[@content="article"]
            - domain: fsb.dossier.center

  - name: dossier_at
    urls:
      - https://www.dossier.at/
      - https://www.dossier.at/dossiers/aktuelles/
      - https://www.dossier.at/dossiers/omv/
      - https://www.dossier.at/dossiers/kronen-zeitung/
    domain_concurrency: 1
    delay: 0.5
    crawl:
      and:
        - or:
            - domain: www.dossier.at
        - not:
            or:
              - mime: nonweb
              - prefix: https://www.dossier.at/asyl/karte
              - prefix: https://www.dossier.at/fileadmin
    parse:
      or:
        - element: .//article[@class="article"]

  - name: daphne_foundation
    urls:
      - https://www.daphne.foundation/en/
      - https://www.daphne.foundation/en/news
    crawl:
      and:
        - or:
            - domain: daphne.foundation
        - not:
            or:
              - prefix: https://www.daphne.foundation/documents/
    parse:
      or:
        - element: .//article

  - name: istories_media
    urls:
      - https://istories.media/
    domain_concurrency: 1
    delay: 10
    crawl:
      and:
        - or:
            - domain: istories.media
        - not:
            or:
              - mime: nonweb
              - domain: static.istories.media
              - domain: zarubezhnye-aktivy.istories.media
    parse:
      or:
        - element: .//div[@class="article-info"]

  - name: daily_maverick
    urls:
      - https://www.dailymaverick.co.za/
      - https://www.dailymaverick.co.za/author/amabhungane/
      - https://www.dailymaverick.co.za/section/scorpio/
      - https://www.dailymaverick.co.za/about/
      - https://www.dailymaverick.co.za/?p=1387616
    domain_concurrency: 1
    delay: 0
    query_ignore:
      - utm_source
      - utm_campaign
      - utm_medium
      - __cf_chl_captcha_tk__
      - __twitter_impression
    crawl:
      and:
        - or:
            - pattern: https://www\.dailymaverick\.co\.za.*
        - not:
            or:
              - mime: nonweb
              # - domain: shop.dailymaverick.co.za
              # - domain: internal.dailymaverick.co.za
              # - domain: local.dailymaverick.co.za
              # - domain: www1.www.dailymaverick.co.za
              # - domain: emma.thewww.dailymaverick.co.za
              # - domain: firstthing.www.dailymaverick.co.za
              # - domain: revengine-tracker.remp.dailymaverick.co.za
              - prefix: https://www.dailymaverick.co.za/opinionista/
              - prefix: https://www.dailymaverick.co.za/opinion-piece
              - prefix: https://www.dailymaverick.co.za/wp-content/
              - prefix: https://www.dailymaverick.co.za/cartoon-tag
              - pattern: .*have-a-picture-for-you.*
    parse:
      or:
        - prefix: https://www.dailymaverick.co.za/article/2022
        - prefix: https://www.dailymaverick.co.za/article/2021
        - prefix: https://www.dailymaverick.co.za/article/2020
        - element: .//div[@class="author-bar"]

  - name: amabhungane
    urls:
      - https://amabhungane.org/
      - https://amabhungane.org/stories/
      - https://amabhungane.org/stories/page/2/
      - https://amabhungane.org/stories/page/3/
      - https://amabhungane.org/stories/page/4/
    domain_concurrency: 2
    delay: 0
    query_ignore:
      - utm_source
      - utm_campaign
      - utm_medium
    crawl:
      and:
        - or:
            - domain: amabhungane.org
        - not:
            or:
              - mime: nonweb
              - prefix: https://amabhungane.org/wp-content/uploads
    parse:
      or:
        - element: .//meta[@content="article"]