add pyproject.toml

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/custom.md

@@ -0,0 +1,10 @@
+---
+name: Custom issue template
+about: Describe this issue template's purpose here.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/dependency-review.yml

@@ -0,0 +1,39 @@
+# Dependency Review Action
+#
+# This Action will scan dependency manifest files that change as part of a Pull Request,
+# surfacing known-vulnerable versions of the packages declared or updated in the PR.
+# Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable
+# packages will be blocked from merging.
+#
+# Source repository: https://github.com/actions/dependency-review-action
+# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
+name: 'Dependency review'
+on:
+  pull_request:
+    branches: [ "main" ]
+
+# If using a dependency submission action in this workflow this permission will need to be set to:
+#
+# permissions:
+#   contents: write
+#
+# https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api
+permissions:
+  contents: read
+  # Write permissions for pull-requests are required for using the `comment-summary-in-pr` option, comment out if you aren't using this option
+  pull-requests: write
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout repository'
+        uses: actions/checkout@v4
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v4
+        # Commonly enabled options, see https://github.com/actions/dependency-review-action#configuration-options for all available options.
+        with:
+          comment-summary-in-pr: always
+        #   fail-on-severity: moderate
+        #   deny-licenses: GPL-1.0-or-later, LGPL-2.0-or-later
+        #   retry-on-snapshot-warnings: true

.github/workflows/pylint.yml

@@ -0,0 +1,26 @@
+on:
+  push:
+    paths:
+      - 'scrapegraphai/**'
+      - '.github/workflows/pylint.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v3
+      - name: Install dependencies
+        run: rye sync --no-lock
+      - name: Analysing the code with pylint
+        run: rye run pylint-ci
+      - name: Check Pylint score
+        run: |
+          pylint_score=$(rye run pylint-score-ci | grep 'Raw metrics' | awk '{print $4}')
+          if (( $(echo "$pylint_score < 8" | bc -l) )); then
+            echo "Pylint score is below 8. Blocking commit."
+            exit 1
+          else
+            echo "Pylint score is acceptable."
+          fi

biblio/extract_references.py

@@ -19,9 +19,9 @@ def extract_references(text, model="gpt-4"):
         model=model,
         messages=[
             {"role": "system", "content": "You are a helpful assistant that extracts references from text."},
-            {"role": "user", "content": f"Extract all references from the following text and format them in a 
+            {"role": "user", "content": f"""Extract all references from the following text and format them in a 
              consistent manner:\n\n{text}\n\nFormat each reference as:\n1. \"Title\" - [Reference Number]\n2. 
-             \"Title\" - [Reference Number]\n..."}
+             \"Title\" - [Reference Number]\n..."""}
         ],
         max_tokens=4096,
         n=1,

pyproject.toml

@@ -0,0 +1,71 @@
+[project]
+name = "Scrape-schema"
+version = "0.0.1"
+description = "library for creating ontologies from documents"
+authors = [
+    { name = "Marco Vinciguerra", email = "[email protected]" },
+    { name = "Marco Perini", email = "[email protected]" },
+    { name = "Lorenzo Padoan", email = "[email protected]" }
+]
+
+dependencies = [
+    "openai==0.28.0",
+    "PyPDF2==3.0.1",
+    "requests==2.32.3",
+    "scrapegraphai==1.18.1"
+]
+
+license = "MIT"
+readme = "README.md"
+homepage = "https://scrapegraphai.com/"
+repository = ""
+documentation = ""
+keywords = [
+    "scrapegraph",
+    "scrapegraphai",
+    "langchain",
+    "ai",
+    "artificial intelligence",
+    "gpt",
+    "machine learning",
+    "rag",
+    "nlp",
+    "natural language processing",
+    "openai",
+    "scraping",
+    "web scraping",
+    "web scraping library",
+    "web scraping tool",
+    "webscraping",
+    "graph",
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.9,<4.0"
+
+[project.optional-dependencies]
+burr = ["burr[start]==0.22.1"]
+docs = ["sphinx==6.0", "furo==2024.5.6"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "pytest==8.0.0",
+    "pytest-mock==3.14.0",
+    "-e file:.[burr]",
+    "-e file:.[docs]",
+    "pylint>=3.2.5",
+]
+
+[tool.rye.scripts]
+pylint-local = "pylint scrapegraphai/**/*.py"
+pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+update-requirements = "python 'manual deployment/autorequirements.py'"

requirements.txt

@@ -0,0 +1,4 @@
+openai==0.28.0
+PyPDF2==3.0.1
+Requests==2.32.3
+scrapegraphai==1.18.1