ci-cd.yaml

# Copyright Paion Data
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
name: CI/CD

"on":
  pull_request:
  push:
    branches:
      - master

env:
  PYTHON_VERSION: "3.10"

jobs:
  cancel-previous:
    name: Cancel Previous Runs In Order to Allocate Action Resources Immediately for Current Run
    if: github.ref != 'refs/heads/master'
    runs-on: ubuntu-latest
    steps:
      - name: Cancel previous
        uses: styfle/cancel-workflow-action@0.10.1
        with:
          access_token: ${{ github.token }}

  yaml-lint:
    name: YAML Style Check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actionshub/yamllint@main

  linkChecker:
    name: Link Check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Link Checker
        uses: lycheeverse/lychee-action@v1.9.0
        with:
          fail: true

  sync-to-huggingface-space:
    needs: [yaml-lint, linkChecker]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Download raw wiktionary data
        run: wget https://kaikki.org/dictionary/raw-wiktextract-data.jsonl
      - name: Install dependencies
        run: pip3 install -r requirements.txt
      - name: Process raw-wiktextract-data.jsonl
        run: |
          python3 ./extract.py -i raw-wiktextract-data.jsonl
          rm raw-wiktextract-data.jsonl
      - name: Push to hub
        if: github.ref == 'refs/heads/master'
        run: |
          git config --global user.name "QubitPi"
          git config --global user.email "jack20220723@gmail.com"

          git lfs install
          git lfs track "*-wiktextract-data.jsonl"
          git lfs track "word-definition-graph-data.jsonl"

          git add *-wiktextract-data.jsonl
          git add word-definition-graph-data.jsonl

          git commit -m "Extract raw-wiktextract-data.jsonl into per-language wiktextract-data.jsonl and generate graph data"
          git push https://paion-data:$HF_TOKEN@huggingface.co/datasets/paion-data/wiktionary-data master:main -f
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}