initial

.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI
+
+on: [push, pull_request, ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [3.8, 3.9, "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox and any other packages
+        run: pip install tox tox-gh-actions
+      - name: Run Tox
+        # Run tox using the version of Python in `PATH`
+        # run: tox -e py${{ matrix.python }}-${{ matrix.django }}
+        run: tox
+
+  coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install flake8
+        run: pip install tox
+      - name: Run tox with coverage
+        # Run tox using the version of Python in `PATH`
+        # run: tox -e py${{ matrix.python }}-${{ matrix.django }}
+        run: tox -e py39-django32-coverage
+
+
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install flake8
+        run: pip install flake8
+      - name: Run flake8
+        # Run tox using the version of Python in `PATH`
+        # run: tox -e py${{ matrix.python }}-${{ matrix.django }}
+        run: flake8

.github/workflows/release.yml

@@ -0,0 +1,28 @@
+name: Release on PYPI
+on:
+  push:
+    branches:  [main]
+
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'bnzk/django-postgres-searchindex'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade setuptools wheel twine
+    - name: Build
+      run: |
+        python setup.py sdist
+        python setup.py bdist_wheel --universal
+    - name: Publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USER_BNZK }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PW_BNZK }}
+      run: twine upload dist/*

.gitignore

@@ -0,0 +1,63 @@
+.idea/
+*.egg-info/
+*.pyc
+*.coverage
+*coverage/
+db.sqlite
+dist/
+docs/_build/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+

AUTHORS

@@ -0,0 +1,7 @@
+Current or previous core committers
+
+Ben Stähli
+
+Contributors (in alphabetical order)
+
+* Your name could stand here :)

CHANGELOG.txt

@@ -0,0 +1,2 @@
+=== (ongoing) ===
+

DESCRIPTION

@@ -0,0 +1,2 @@
+Everything in postgres, accessible via Django ORM, using
+postgres fullext search capabilites

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Ben Stähli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,10 @@
+include AUTHORS
+include LICENSE
+include DESCRIPTION
+include CHANGELOG.txt
+include README.md
+graft postgres_searchindex
+global-exclude __pycache__
+global-exclude *.orig *.pyc *.log *.swp
+prune postgres_searchindex/tests/coverage
+prune postgres_searchindex/.ropeproject

README.md

@@ -0,0 +1,127 @@
+# django-postgres-searchindex
+
+[![CI](https://img.shields.io/github/actions/workflow/status/bnzk/django-postgres-searchindex/ci.yml?style=flat-square&logo=github "CI")](https://github.com/bnzk/django-postgres-searchindex/actions/workflows/ci.yml)
+[![Version](https://img.shields.io/pypi/v/django-postgres-searchindex.svg?style=flat-square "Version")](https://pypi.python.org/pypi/django-postgres-searchindex/)
+[![Licence](https://img.shields.io/github/license/bnzk/django-postgres-searchindex.svg?style=flat-square "Licence")](https://pypi.python.org/pypi/django-postgres-searchindex/)
+[![PyPI Downloads](https://img.shields.io/pypi/dm/django-postgres-searchindex?style=flat-square "PyPi Downloads")](https://pypistats.org/packages/django-postgres-searchindex)
+
+A bit like django-haystack, but everything in postgres, accessible via Django ORM, using
+postgres fullext search capabilites. The goal is to ease setup and 
+maintainance for smaller and medium sized projects - without dependencies on
+search technology like elastic, solr or whoosh.
+
+During conception, I was thinking about developing a backend for django-haystack, but 
+decided against, to be able to develop from the ground up, as simple as possible. The 
+project could still provide a haystack backend one day, but it was just not my priority.
+
+## Quickstart
+
+Describe, index, search.
+
+### Define index(es) in django settings
+
+Default value, simplest possible configuration:
+
+```
+POSTGRES_SEARCHINDEX = {
+    "default": {},
+}
+```
+
+Example for a multilanguage setup:
+
+```
+POSTGRES_SEARCHINDEX = {
+    "de": {
+        "kwargs": {
+            "language": "de",
+        }
+    },
+    "fr": {
+        "kwargs": {
+            "language": "fr",
+        }
+    },
+}
+```
+
+More complex configurations could include django's `SITE_ID` or other relevant infos
+in searchindex key and kwargs.
+
+### Define sources
+
+Example, hopefully self explaining.
+
+```
+from django.utils.html import strip_tags
+from postgres_searchindex.base import IndexSource / MultiLanguageIndexSource
+from postgres_searchindex.source_pool import source_pool
+
+from news.models import News
+
+
+@source_pool.register
+class NewsIndexSource(IndexSource / MultiLanguageIndexSource):
+    model = News
+
+    def get_title(self, obj):
+        return strip_tags(obj.description)
+
+    def get_content(self, obj):
+        return strip_tags(obj.description)
+
+    def get_queryset(self):
+        return self.model.objects.published()
+```
+
+### Populate the index
+
+Run  `./manage.py postgres_searchindex_update` to update/build the index.
+
+If you want to control how things get indexed, you can check
+your `IndexEntry` instances in Django admin.
+
+### Search!
+
+You can now search in your index. You are free to use [Django's builtin fulltext](https://docs.djangoproject.com/en/dev/ref/contrib/postgres/search/)
+features as you like - as in the following example, or in a way more advanced manner.
+
+```
+from django.contrib.postgres.search import SearchVector
+from postgres_searchindex.models import IndexEntry
+
+# this will return entries containing "überhaupt" and "uberhaupt"
+IndexEntry.objects.annotate(
+    search=SearchVector("content", "title", config="german")
+).filter(index_key=self.request.LANGUAGE_CODE, search="uberhaupt")
+
+```
+
+There is a full example in the source: `views.py` and `urls.py` will give you an idea. 
+
+To be done: |highlight:query templatefilter, to highlight the serach query in the 
+search result text.
+
+### Keep the index fresh
+
+Either you'll regularly run `./manage.py postgres_searchindex_update`, or you'll 
+implement a realtime or near realtime solution, with signals, throug the 
+`POSTGRES_SEARCHINDEX_SIGNAL_PROCESSOR` setting. 
+
+There are ~~two~~ currently one builtin processors:
+ - `postgres_searchindex.signal_processors.RealtimeSyncedSignalProcessor`
+ - To be done! `postgres_searchindex.signal_processors.RealtimeCelerySignalProcessor`
+
+The async signal processor will require you to have celery configured.
+
+
+## TODO
+
+- properly handle removal of instances
+- use trigram search?
+- create an index for `content` and title
+- instant update index via signals (update/delete models)
+  - with celery? 
+  - https://www.world-of-knives.ch/de/messershop/m-88-acier-japonais/
+  - manage command: INdexEntry.objects.filter(original=None).delete() doesnt work?
+    check each model...

manage.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE',
+                          # 'postgres_searchindex.tests.south_settings')
+                          'postgres_searchindex.tests.settings')
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

postgres_searchindex/admin.py

@@ -0,0 +1,19 @@
+from django.contrib import admin
+
+from .models import IndexEntry
+
+
+@admin.register(IndexEntry)
+class IndexEntryAdmin(admin.ModelAdmin):
+    list_display_links = ("title",)
+    list_display = (
+        "index_key",
+        "title",
+        "url",
+        "modified_at",
+    )
+    search_fields = ("title", "content")
+    list_filter = (
+        "index_key",
+        "content_type",
+    )

postgres_searchindex/apps.py

@@ -0,0 +1,11 @@
+from django.apps import AppConfig
+
+
+class PostgresSearchIndexConfig(AppConfig):
+    name = "postgres_searchindex"
+
+    def ready(self):
+        # discover all search indexes.
+        from postgres_searchindex.source_pool import source_pool
+
+        source_pool.discover()