diff --git a/README.md b/README.md index d32d20d5..00acf97d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ |----------|-----------|-----|--------|-------|--------|----| | Go | Echo | ✅ | ✅ | ✅ | ✅ | X | | Go | Gin | ✅ | ✅ | ✅ | ✅ | X | -| Python | Django | ✅ | X | X | X | X | +| Python | Django | ✅ | ✅ | ✅ | ✅ | X | | Python | Flask | ✅ | X | X | X | X | | Ruby | Rails | ✅ | ✅ | ✅ | ✅ | X | | Ruby | Sinatra | ✅ | ✅ | ✅ | ✅ | X | diff --git a/spec/functional_test/fixtures/django/README.md b/spec/functional_test/fixtures/django/README.md new file mode 100644 index 00000000..e6d2a4de --- /dev/null +++ b/spec/functional_test/fixtures/django/README.md @@ -0,0 +1,3 @@ +## Notes + +This project uses a portion of the source code from [DjangoBlog](https://github.com/liangliangyy/DjangoBlog) for testing purposes. (Djangoblog is released under the MIT License) \ No newline at end of file diff --git a/spec/functional_test/fixtures/django/accounts/urls.py b/spec/functional_test/fixtures/django/accounts/urls.py deleted file mode 100644 index 3c4d3153..00000000 --- a/spec/functional_test/fixtures/django/accounts/urls.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.urls import path -from registration.backends.default.views import RegistrationView -from registration.forms import RegistrationFormUniqueEmail - -from . import views as account_views - -urlpatterns = [ - path( - "register/", - RegistrationView.as_view(form_class=RegistrationFormUniqueEmail), - name="registration_register", - ), - path( - "edit/", - account_views.edit_profile, - name="edit_profile", - ), -] diff --git a/spec/functional_test/fixtures/django/blog/__init__.py b/spec/functional_test/fixtures/django/blog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/django/blog/urls.py b/spec/functional_test/fixtures/django/blog/urls.py new file mode 100644 index 00000000..bd75633c --- /dev/null +++ b/spec/functional_test/fixtures/django/blog/urls.py @@ -0,0 +1,70 @@ +from django.urls import path +from django.views.decorators.cache import cache_page + +from . import views + +app_name = "blog" +urlpatterns = [ + path( + r'', + views.IndexView.as_view(), + name='index'), + path( + r'page//', + views.IndexView.as_view(), + name='index_page'), + path( + r'article////.html', + views.ArticleDetailView.as_view(), + name='detailbyid'), + path( + r'category/.html', + views.CategoryDetailView.as_view(), + name='category_detail'), + path( + r'category//.html', + views.CategoryDetailView.as_view(), + name='category_detail_page'), + path( + r'author/.html', + views.AuthorDetailView.as_view(), + name='author_detail'), + path( + r'author//.html', + views.AuthorDetailView.as_view(), + name='author_detail_page'), + path( + r'tag/.html', + views.TagDetailView.as_view(), + name='tag_detail'), + path( + r'tag//.html', + views.TagDetailView.as_view(), + name='tag_detail_page'), + path( + 'archives.html', + cache_page( + 60 * 60)( + views.ArchivesView.as_view()), + name='archives'), + path( + 'links.html', + views.LinkListView.as_view(), + name='links'), + path( + r'upload', + views.fileupload, + name='upload'), + path( + r'not_found', + views.page_not_found_view, + name='page_not_found_view'), + path( + r'test', + views.test, + name='test'), + path( + r'delete_test', + views.delete_test, + name='delete_test'), +] diff --git a/spec/functional_test/fixtures/django/blog/views.py b/spec/functional_test/fixtures/django/blog/views.py new file mode 100644 index 00000000..8821314b --- /dev/null +++ b/spec/functional_test/fixtures/django/blog/views.py @@ -0,0 +1,395 @@ +import logging +import os +import uuid + +from django.conf import settings +from django.core.paginator import Paginator +from django.http import HttpResponse, HttpResponseForbidden +from django.shortcuts import get_object_or_404 +from django.shortcuts import render +from django.templatetags.static import static +from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt +from django.views.generic.detail import DetailView +from django.views.generic.list import ListView +from haystack.views import SearchView + +from blog.models import Article, Category, LinkShowType, Links, Tag +from comments.forms import CommentForm +from djangoblog.utils import cache, get_blog_setting, get_sha256 + +logger = logging.getLogger(__name__) + + +class ArticleListView(ListView): + # template_name属性用于指定使用哪个模板进行渲染 + template_name = 'blog/article_index.html' + + # context_object_name属性用于给上下文变量取名（在模板中使用该名字） + context_object_name = 'article_list' + + # 页面类型，分类目录或标签列表等 + page_type = '' + paginate_by = settings.PAGINATE_BY + page_kwarg = 'page' + link_type = LinkShowType.L + + def get_view_cache_key(self): + return self.request.get['pages'] + + @property + def page_number(self): + page_kwarg = self.page_kwarg + page = self.kwargs.get( + page_kwarg) or self.request.GET.get(page_kwarg) or 1 + return page + + def get_queryset_cache_key(self): + """ + 子类重写.获得queryset的缓存key + """ + raise NotImplementedError() + + def get_queryset_data(self): + """ + 子类重写.获取queryset的数据 + """ + raise NotImplementedError() + + def get_queryset_from_cache(self, cache_key): + ''' + 缓存页面数据 + :param cache_key: 缓存key + :return: + ''' + value = cache.get(cache_key) + if value: + logger.info('get view cache.key:{key}'.format(key=cache_key)) + return value + else: + article_list = self.get_queryset_data() + cache.set(cache_key, article_list) + logger.info('set view cache.key:{key}'.format(key=cache_key)) + return article_list + + def get_queryset(self): + ''' + 重写默认，从缓存获取数据 + :return: + ''' + key = self.get_queryset_cache_key() + value = self.get_queryset_from_cache(key) + return value + + def get_context_data(self, **kwargs): + kwargs['linktype'] = self.link_type + return super(ArticleListView, self).get_context_data(**kwargs) + + +class IndexView(ArticleListView): + ''' + 首页 + ''' + # 友情链接类型 + link_type = LinkShowType.I + + def get_queryset_data(self): + article_list = Article.objects.filter(type='a', status='p') + return article_list + + def get_queryset_cache_key(self): + cache_key = 'index_{page}'.format(page=self.page_number) + return cache_key + + +class ArticleDetailView(DetailView): + ''' + 文章详情页面 + ''' + template_name = 'blog/article_detail.html' + model = Article + pk_url_kwarg = 'article_id' + context_object_name = "article" + + def get_object(self, queryset=None): + obj = super(ArticleDetailView, self).get_object() + obj.viewed() + self.object = obj + return obj + + def get_context_data(self, **kwargs): + comment_form = CommentForm() + + article_comments = self.object.comment_list() + parent_comments = article_comments.filter(parent_comment=None) + blog_setting = get_blog_setting() + paginator = Paginator(parent_comments, blog_setting.article_comment_count) + page = self.request.GET.get('comment_page', '1') + if not page.isnumeric(): + page = 1 + else: + page = int(page) + if page < 1: + page = 1 + if page > paginator.num_pages: + page = paginator.num_pages + + p_comments = paginator.page(page) + next_page = p_comments.next_page_number() if p_comments.has_next() else None + prev_page = p_comments.previous_page_number() if p_comments.has_previous() else None + + if next_page: + kwargs[ + 'comment_next_page_url'] = self.object.get_absolute_url() + f'?comment_page={next_page}#commentlist-container' + if prev_page: + kwargs[ + 'comment_prev_page_url'] = self.object.get_absolute_url() + f'?comment_page={prev_page}#commentlist-container' + kwargs['form'] = comment_form + kwargs['article_comments'] = article_comments + kwargs['p_comments'] = p_comments + kwargs['comment_count'] = len( + article_comments) if article_comments else 0 + + kwargs['next_article'] = self.object.next_article + kwargs['prev_article'] = self.object.prev_article + + return super(ArticleDetailView, self).get_context_data(**kwargs) + + +class CategoryDetailView(ArticleListView): + ''' + 分类目录列表 + ''' + page_type = "分类目录归档" + + def get_queryset_data(self): + slug = self.kwargs['category_name'] + category = get_object_or_404(Category, slug=slug) + + categoryname = category.name + self.categoryname = categoryname + categorynames = list( + map(lambda c: c.name, category.get_sub_categorys())) + article_list = Article.objects.filter( + category__name__in=categorynames, status='p') + return article_list + + def get_queryset_cache_key(self): + slug = self.kwargs['category_name'] + category = get_object_or_404(Category, slug=slug) + categoryname = category.name + self.categoryname = categoryname + cache_key = 'category_list_{categoryname}_{page}'.format( + categoryname=categoryname, page=self.page_number) + return cache_key + + def get_context_data(self, **kwargs): + + categoryname = self.categoryname + try: + categoryname = categoryname.split('/')[-1] + except BaseException: + pass + kwargs['page_type'] = CategoryDetailView.page_type + kwargs['tag_name'] = categoryname + return super(CategoryDetailView, self).get_context_data(**kwargs) + + +class AuthorDetailView(ArticleListView): + ''' + 作者详情页 + ''' + page_type = '作者文章归档' + + def get_queryset_cache_key(self): + from uuslug import slugify + author_name = slugify(self.kwargs['author_name']) + cache_key = 'author_{author_name}_{page}'.format( + author_name=author_name, page=self.page_number) + return cache_key + + def get_queryset_data(self): + author_name = self.kwargs['author_name'] + article_list = Article.objects.filter( + author__username=author_name, type='a', status='p') + return article_list + + def get_context_data(self, **kwargs): + author_name = self.kwargs['author_name'] + kwargs['page_type'] = AuthorDetailView.page_type + kwargs['tag_name'] = author_name + return super(AuthorDetailView, self).get_context_data(**kwargs) + + +class TagDetailView(ArticleListView): + ''' + 标签列表页面 + ''' + page_type = '分类标签归档' + + def get_queryset_data(self): + slug = self.kwargs['tag_name'] + tag = get_object_or_404(Tag, slug=slug) + tag_name = tag.name + self.name = tag_name + article_list = Article.objects.filter( + tags__name=tag_name, type='a', status='p') + return article_list + + def get_queryset_cache_key(self): + slug = self.kwargs['tag_name'] + tag = get_object_or_404(Tag, slug=slug) + tag_name = tag.name + self.name = tag_name + cache_key = 'tag_{tag_name}_{page}'.format( + tag_name=tag_name, page=self.page_number) + return cache_key + + def get_context_data(self, **kwargs): + # tag_name = self.kwargs['tag_name'] + tag_name = self.name + kwargs['page_type'] = TagDetailView.page_type + kwargs['tag_name'] = tag_name + return super(TagDetailView, self).get_context_data(**kwargs) + + +class ArchivesView(ArticleListView): + ''' + 文章归档页面 + ''' + page_type = '文章归档' + paginate_by = None + page_kwarg = None + template_name = 'blog/article_archives.html' + + def get_queryset_data(self): + return Article.objects.filter(status='p').all() + + def get_queryset_cache_key(self): + cache_key = 'archives' + return cache_key + + +class LinkListView(ListView): + model = Links + template_name = 'blog/links_list.html' + + def get_queryset(self): + return Links.objects.filter(is_enable=True) + + +class EsSearchView(SearchView): + def get_context(self): + paginator, page = self.build_page() + context = { + "query": self.query, + "form": self.form, + "page": page, + "paginator": paginator, + "suggestion": None, + } + if hasattr(self.results, "query") and self.results.query.backend.include_spelling: + context["suggestion"] = self.results.query.get_spelling_suggestion() + context.update(self.extra_context()) + + return context + + +@csrf_exempt +def fileupload(request): + """ + 该方法需自己写调用端来上传图片，该方法仅提供图床功能 + :param request: + :return: + """ + status = request.META.get('HTTP_X_FORWARDED_FOR') == '127.0.0.1' + status = status and request.META.get('X_REAL_IP') == '127.0.0.1' + + if request.method == 'POST': + sign = request.GET.get('sign', None) + if not sign: + return HttpResponseForbidden() + if not sign == get_sha256(get_sha256(settings.SECRET_KEY)): + return HttpResponseForbidden() + + if not status: + return "is not localhost" + + response = [] + for filename in request.FILES: + timestr = timezone.now().strftime('%Y/%m/%d') + imgextensions = ['jpg', 'png', 'jpeg', 'bmp'] + fname = u''.join(str(filename)) + isimage = len([i for i in imgextensions if fname.find(i) >= 0]) > 0 + base_dir = os.path.join(settings.STATICFILES, "files" if not isimage else "image", timestr) + if not os.path.exists(base_dir): + os.makedirs(base_dir) + savepath = os.path.normpath(os.path.join(base_dir, f"{uuid.uuid4().hex}{os.path.splitext(filename)[-1]}")) + if not savepath.startswith(base_dir): + return HttpResponse("only for post") + with open(savepath, 'wb+') as wfile: + for chunk in request.FILES[filename].chunks(): + wfile.write(chunk) + if isimage: + from PIL import Image + image = Image.open(savepath) + image.save(savepath, quality=20, optimize=True) + url = static(savepath) + response.append(url) + return HttpResponse(response) + + else: + return HttpResponse("only for post") + + +def page_not_found_view(request, + exception=None, + template_name='blog/error_page.html'): + if exception: + logger.error(exception) + url = request.get_full_path() + + dummpy_cookie = '' + if request.COOKIES['app_type'] == 'ksg97031': + dummpy_cookie = 'ksg97031' + + + return render(request, + template_name, + {'message': '哎呀，您访问的地址 ' + url + ' 是一个未知的地方。请点击首页看看别的？', + 'statuscode': '404'}, + status=404) + +def test(request): + if request.method == 'POST': + return request.data.get('test_param', 'no params') + if request.method == 'PUT': + return request.data.get('test_param', 'no params') + if request.method == 'PATCH': + return request.data.get('test_param', 'no params') + + return "test" + +def delete_test(request): + if request.method == 'DELETE': + return 'delete' + + return "test" + +def server_error_view(request, template_name='blog/error_page.html'): + return render(request, + template_name, + {'message': '哎呀，出错了，我已经收集到了错误信息，之后会抓紧抢修，请点击首页看看别的？', + 'statuscode': '500'}, + status=500) + + +def permission_denied_view( + request, + exception, + template_name='blog/error_page.html'): + if exception: + logger.error(exception) + return render( + request, template_name, { + 'message': '哎呀，您没有权限访问此页面，请点击首页看看别的？', 'statuscode': '403'}, status=403) diff --git a/spec/functional_test/fixtures/django/djangoblog/__init__.py b/spec/functional_test/fixtures/django/djangoblog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/django/djangoblog/settings.py b/spec/functional_test/fixtures/django/djangoblog/settings.py new file mode 100644 index 00000000..e64f3a24 --- /dev/null +++ b/spec/functional_test/fixtures/django/djangoblog/settings.py @@ -0,0 +1,81 @@ +""" +Django settings for djangoblog project. + +Generated by 'django-admin startproject' using Django 1.10.2. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.10/ref/settings/ +""" +import os +import sys + + +def env_to_bool(env, default): + str_val = os.environ.get(env) + return default if str_val is None else str_val == 'True' + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get( + 'DJANGO_SECRET_KEY') or 'n9ceqv38)#&mwuat@(mjb_p%em$e8$qyr#fw9ot!=ba6lijx-6' +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = env_to_bool('DJANGO_DEBUG', True) +# DEBUG = False +TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test' + +# ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*', '127.0.0.1', 'example.com'] +# django 4.0新增配置 +CSRF_TRUSTED_ORIGINS = ['http://example.com'] +# Application definition + + +INSTALLED_APPS = [ + # 'django.contrib.admin', + 'django.contrib.admin.apps.SimpleAdminConfig', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'django.contrib.sitemaps', + 'mdeditor', + 'haystack', + 'blog', + 'accounts', + 'comments', + 'oauth', + 'servermanager', + 'owntracks', + 'compressor' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.gzip.GZipMiddleware', + # 'django.middleware.cache.UpdateCacheMiddleware', + 'django.middleware.common.CommonMiddleware', + # 'django.middleware.cache.FetchFromCacheMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.http.ConditionalGetMiddleware', + 'blog.middleware.OnlineMiddleware' +] + +ROOT_URLCONF = 'djangoblog.urls' + +# [REDACTED] diff --git a/spec/functional_test/fixtures/django/djangoblog/urls.py b/spec/functional_test/fixtures/django/djangoblog/urls.py new file mode 100644 index 00000000..cfd94b10 --- /dev/null +++ b/spec/functional_test/fixtures/django/djangoblog/urls.py @@ -0,0 +1,47 @@ +"""djangoblog URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf import settings +from django.conf.urls.static import static +from django.contrib.sitemaps.views import sitemap +from django.urls import include +from django.urls import re_path +from haystack.views import search_view_factory + +from blog.views import EsSearchView +from djangoblog.admin_site import admin_site +from djangoblog.elasticsearch_backend import ElasticSearchModelSearchForm +from djangoblog.feeds import DjangoBlogFeed +from djangoblog.sitemap import ArticleSiteMap, CategorySiteMap, StaticViewSitemap, TagSiteMap, UserSiteMap + +sitemaps = { + + 'blog': ArticleSiteMap, + 'Category': CategorySiteMap, + 'Tag': TagSiteMap, + 'User': UserSiteMap, + 'static': StaticViewSitemap +} + +handler404 = 'blog.views.page_not_found_view' +handler500 = 'blog.views.server_error_view' +handle403 = 'blog.views.permission_denied_view' +urlpatterns = [ + re_path(r'', include('blog.urls', namespace='blog')), + # [REDACTED] + ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, + document_root=settings.MEDIA_ROOT) diff --git a/spec/functional_test/fixtures/django/djangoproject/settings/common.py b/spec/functional_test/fixtures/django/djangoproject/settings/common.py deleted file mode 100644 index 941129e2..00000000 --- a/spec/functional_test/fixtures/django/djangoproject/settings/common.py +++ /dev/null @@ -1,5 +0,0 @@ -# [REDACTED] - -ROOT_URLCONF = "djangoproject.urls.www" - -# [REDACTED] \ No newline at end of file diff --git a/spec/functional_test/fixtures/django/djangoproject/urls/www.py b/spec/functional_test/fixtures/django/djangoproject/urls/www.py deleted file mode 100644 index 7df28b6f..00000000 --- a/spec/functional_test/fixtures/django/djangoproject/urls/www.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.urls import include, path -from django.views.generic import RedirectView, TemplateView - -urlpatterns = [ - path( - "start/overview/", - TemplateView.as_view(template_name="overview.html"), - name="overview", - ), - path("overview/", RedirectView.as_view(url="/start/overview/", permanent=False)), - # include - path("accounts/", include("accounts.urls")), -] diff --git a/spec/functional_test/fixtures/django/manage.py b/spec/functional_test/fixtures/django/manage.py index 6d406775..919ba740 100755 --- a/spec/functional_test/fixtures/django/manage.py +++ b/spec/functional_test/fixtures/django/manage.py @@ -1,22 +1,22 @@ #!/usr/bin/env python -"""Django's command-line utility for administrative tasks.""" import os import sys - -def main(): - """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoproject.settings.dev") +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoblog.settings") try: from django.core.management import execute_from_command_line - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise execute_from_command_line(sys.argv) - - -if __name__ == "__main__": - main() diff --git a/spec/functional_test/testers/python_django_spec.cr b/spec/functional_test/testers/python_django_spec.cr index 39739b7d..0406ec1f 100644 --- a/spec/functional_test/testers/python_django_spec.cr +++ b/spec/functional_test/testers/python_django_spec.cr @@ -1,16 +1,29 @@ require "../func_spec.cr" extected_endpoints = [ - # djangoproject/urls/www.py - Endpoint.new("/start/overview/", "GET"), - Endpoint.new("/overview/", "GET"), - # accounts/urls.py - Endpoint.new("/accounts/register/", "GET"), - Endpoint.new("/accounts/edit/", "GET"), - Endpoint.new("/accounts/", "GET"), + Endpoint.new("/", "GET"), + Endpoint.new("/page//", "GET"), + Endpoint.new("/article////.html", "GET", [Param.new("comment_page", "", "query")]), + Endpoint.new("/category/.html", "GET"), + Endpoint.new("/category//.html", "GET"), + Endpoint.new("/author/.html", "GET"), + Endpoint.new("/author//.html", "GET"), + Endpoint.new("/tag/.html", "GET"), + Endpoint.new("/tag//.html", "GET"), + Endpoint.new("/archives.html", "GET"), + Endpoint.new("/links.html", "GET"), + Endpoint.new("/upload", "GET", [Param.new("sign", "", "query"), Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), + Endpoint.new("/upload", "POST", [Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), + Endpoint.new("/not_found", "GET", [Param.new("Cookie['app_type']", "", "header")]), + Endpoint.new("/test", "GET", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "POST", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "PUT", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "PATCH", [Param.new("test_param", "", "form")]), + Endpoint.new("/delete_test", "GET"), + Endpoint.new("/delete_test", "DELETE"), ] FunctionalTester.new("fixtures/django/", { :techs => 1, - :endpoints => 5, + :endpoints => 20, }, extected_endpoints).test_all diff --git a/spec/unit_test/analyzer/analyzer_django_spec.cr b/spec/unit_test/analyzer/analyzer_django_spec.cr deleted file mode 100644 index 7440a1f0..00000000 --- a/spec/unit_test/analyzer/analyzer_django_spec.cr +++ /dev/null @@ -1,23 +0,0 @@ -require "../../../src/analyzer/analyzers/analyzer_django.cr" -require "../../../src/options" - -describe "mapping_to_path" do - options = default_options() - instance = AnalyzerDjango.new(options) - - it "mapping_to_path - code style1" do - instance.mapping_to_path("path('home/', views.home_view, name='home'),").should eq(["/home/"]) - end - - it "mapping_to_path - code style2" do - instance.mapping_to_path("path('articles//', views.article_detail_view, name='article_detail'),").should eq(["/articles//"]) - end - - it "mapping_to_path - code style3 (regex)" do - instance.mapping_to_path("re_path(r'^archive/(?Pd{4})/$', views.archive_year_view, name='archive_year'),").should eq(["/archive/(?Pd{4})/"]) - end - - it "mapping_to_path - code style4 (register)" do - instance.mapping_to_path("router.register(r'articles', ArticleViewSet)").should eq(["/articles"]) - end -end diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index 18a4f517..ce8144ea 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -2,10 +2,25 @@ require "../../models/analyzer" require "json" class AnalyzerDjango < Analyzer - REGEX_ROOT_URLCONF = /\s*ROOT_URLCONF\s*=\s*r?['"]([^'"\\]*)['"]/ - REGEX_URL_PATTERNS = /urlpatterns\s*=\s*\[(.*)\]/m - REGEX_URL_MAPPING = /(?:url|path|register)\s*\(\s*r?['"]([^"']*)['"]\s*,\s*([^),]*)/ - REGEX_INCLUDE_URLS = /include\s*\(\s*r?['"]([^'"\\]*)['"]/ + @django_base_path : String = "" + REGEX_ROOT_URLCONF = /\s*ROOT_URLCONF\s*=\s*r?['"]([^'"\\]*)['"]/ + REGEX_ROUTE_MAPPING = /(?:url|path|register)\s*\(\s*r?['"]([^"']*)['"][^,]*,\s*([^),]*)/ + REGEX_INCLUDE_URLS = /include\s*\(\s*r?['"]([^'"\\]*)['"]/ + INDENT_SPACE_SIZE = 4 # Different indentation sizes can result in code analysis being disregarded + HTTP_METHOD_NAMES = ["get", "post", "put", "patch", "delete", "head", "options", "trace"] + REQUEST_PARAM_FIELD_MAP = { + "GET" => {["GET"], "query"}, + "POST" => {["POST"], "form"}, + "COOKIES" => {nil, "header"}, + "META" => {nil, "header"}, + "data" => {["POST", "PUT", "PATCH"], "form"}, + } + REQUEST_PARAM_TYPE_MAP = { + "query" => ["GET"], + "form" => ["GET", "POST", "PUT", "PATCH"], + "cookie" => nil, + "header" => nil, + } def analyze result = [] of Endpoint @@ -13,6 +28,7 @@ class AnalyzerDjango < Analyzer # Django urls root_django_urls_list = search_root_django_urls_list() root_django_urls_list.each do |root_django_urls| + @django_base_path = root_django_urls.basepath get_endpoints(root_django_urls).each do |endpoint| result << endpoint end @@ -30,7 +46,9 @@ class AnalyzerDjango < Analyzer def search_root_django_urls_list : Array(DjangoUrls) root_django_urls_list = [] of DjangoUrls - Dir.glob("#{base_path}/**/*") do |file| + + search_dir = @base_path + Dir.glob("#{search_dir}/**/*") do |file| spawn do begin next if File.directory?(file) @@ -38,9 +56,11 @@ class AnalyzerDjango < Analyzer content = File.read(file, encoding: "utf-8", invalid: :skip) content.scan(REGEX_ROOT_URLCONF) do |match| next if match.size != 2 - filepath = "#{base_path}/#{match[1].gsub(".", "/")}.py" - if File.exists? filepath - root_django_urls_list << DjangoUrls.new("", filepath) + dotted_as_urlconf = match[1].split(".") + relative_path = "#{dotted_as_urlconf.join("/")}.py" + Dir.glob("#{search_dir}/**/#{relative_path}") do |filepath| + basepath = filepath.split("/")[..-(dotted_as_urlconf.size + 1)].join("/") + root_django_urls_list << DjangoUrls.new("", filepath, basepath) end end end @@ -54,64 +74,478 @@ class AnalyzerDjango < Analyzer root_django_urls_list.uniq end + module PackageType + FILE = 0 + CODE = 1 + end + + def travel_package(package_path, dotted_as_names) + travel_package_map = Array(Tuple(String, String, Int32)).new + + py_path = "" + is_travel_positive = false + dotted_as_names_split = dotted_as_names.split(".") + dotted_as_names_split[0..-1].each_with_index do |names, index| + travel_package_path = File.join(package_path, names) + + py_guess = "#{travel_package_path}.py" + if File.directory? travel_package_path + package_path = travel_package_path + is_travel_positive = true + elsif dotted_as_names_split.size - 2 <= index && File.exists? py_guess + py_path = py_guess + is_travel_positive = true + else + break + end + end + + if is_travel_positive == false + return travel_package_map + end + + names = dotted_as_names_split[-1] + names.split(",").each do |name| + _import = name.strip + next if _import == "" + + _alias = nil + if _import.includes? " as " + _import, _alias = _import.split(" as ") + end + + package_type = PackageType::CODE + py_guess = File.join(package_path, "#{_import}.py") + if File.exists? py_guess + package_type = PackageType::FILE + py_path = py_guess + end + + next if py_path == "" + if !_alias.nil? + travel_package_map << {_alias, py_path, package_type} + else + travel_package_map << {_import, py_path, package_type} + end + end + + travel_package_map + end + + def parse_import_packages(url_base_path : String, content : String) + # https://docs.python.org/3/reference/import.html + package_map = {} of String => Tuple(String, Int32) + + offset = 0 + content.each_line do |line| + package_path = @django_base_path + + _from = "" + _imports = "" + _aliases = "" + if line.starts_with? "from" + line.scan(/from\s*([^'"\s\\]*)\s*import\s*(.*)/) do |match| + next if match.size != 3 + _from = match[1] + _imports = match[2] + end + elsif line.starts_with? "import" + line.scan(/import\s*([^'"\s\\]*)/) do |match| + next if match.size != 2 + _imports = match[1] + end + end + + unless _imports == "" + round_bracket_index = line.index('(') + if !round_bracket_index.nil? + # Parse 'import (\n a,\n b,\n c)' pattern + index = offset + round_bracket_index + 1 + while index < content.size && content[index] != ')' + index += 1 + end + _imports = content[(offset + round_bracket_index + 1)..(index - 1)].strip + end + + # Relative path + if _from.starts_with? ".." + package_path = File.join(url_base_path, "..") + _from = _from[2..] + elsif _from.starts_with? "." + package_path = url_base_path + _from = _from[1..] + end + + _imports.split(",").each do |_import| + if _import.starts_with? ".." + package_path = File.join(url_base_path, "..") + elsif _import.starts_with? "." + package_path = url_base_path + end + + dotted_as_names = _import + if _from != "" + dotted_as_names = _from + "." + _import + end + + # Create package map (Hash[name => filepath, ...]) + travel_package_map = travel_package(package_path, dotted_as_names) + next if travel_package_map.size == 0 + travel_package_map.each do |travel_package| + name, filepath, package_type = travel_package + package_map[name] = {filepath, package_type} + end + end + end + + offset += line.size + 1 + end + + package_map + end + def get_endpoints(django_urls : DjangoUrls) : Array(Endpoint) endpoints = [] of Endpoint - paths = get_paths(django_urls) - paths.each do |path| - path = path.gsub("//", "/") - unless path.starts_with?("/") - path = "/#{path}" + url_base_path = File.dirname(django_urls.filepath) + + file = File.open(django_urls.filepath, encoding: "utf-8", invalid: :skip) + content = file.gets_to_end + package_map = parse_import_packages(url_base_path, content) + + # [Temporary Fix] Parse only the string after "urlpatterns = [" + keywords = ["urlpatterns", "=", "["] + keywords.each do |keyword| + if !content.includes? keyword + return endpoints + end + + content = content.split(keyword, 2)[1] + end + + # [TODO] Parse correct urlpatterns from variable concatenation case" + content.scan(REGEX_ROUTE_MAPPING) do |route_match| + next if route_match.size != 3 + route = route_match[1] + route = route.gsub(/^\^/, "").gsub(/\$$/, "") + view = route_match[2].split(",")[0] + url = "/#{@url}/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/") + + new_django_urls = nil + view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match| + # Detect new url configs + next if include_pattern_match.size != 2 + new_route_path = "#{@django_base_path}/#{include_pattern_match[1].gsub(".", "/")}.py" + + if File.exists?(new_route_path) + new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath) + get_endpoints(new_django_urls).each do |endpoint| + endpoints << endpoint + end + end end + next if new_django_urls != nil - endpoints << Endpoint.new("#{@url}#{path}", "GET") + if view == "" + endpoints << Endpoint.new(url, "GET") + else + dotted_as_names_split = view.split(".") + + filepath = "" + function_or_class_name = "" + dotted_as_names_split.each_with_index do |name, index| + if (package_map.has_key? name) && (index < dotted_as_names_split.size) + filepath, package_type = package_map[name] + function_or_class_name = name + if package_type == PackageType::FILE && index + 1 < dotted_as_names_split.size + function_or_class_name = dotted_as_names_split[index + 1] + end + + break + end + end + + if filepath != "" + get_endpoint_from_files(url, filepath, function_or_class_name).each do |endpoint| + endpoints << endpoint + end + else + # By default, Django allows requests with methods other than GET as well + # Prevent this flow, we need to improve trace code of 'get_endpoint_from_files() + endpoints << Endpoint.new(url, "GET") + end + end end endpoints end - def get_paths(django_urls : DjangoUrls) - paths = [] of String - content = File.read(django_urls.filepath, encoding: "utf-8", invalid: :skip) - content.scan(REGEX_URL_PATTERNS) do |match| - next if match.size != 2 - paths = mapping_to_path(match[1], django_urls.prefix) + def get_endpoint_from_files(url : String, filepath : String, function_or_class_name : String) + endpoints = Array(Endpoint).new + suspicious_http_methods = ["GET"] + suspicious_params = Array(Param).new + + content = File.read(filepath, encoding: "utf-8", invalid: :skip) + content_lines = content.split "\n" + + # Function Based View + function_start_index = content.index /def\s+#{function_or_class_name}\s*\(/ + if !function_start_index.nil? + function_codeblock = parse_function_or_class(content[function_start_index..]) + if !function_codeblock.nil? + lines = function_codeblock.split "\n" + function_define_line = lines[0] + lines = lines[1..] + + # Verify if the decorator line contains an HTTP method, for instance: + # '@api_view(['POST'])', '@require_POST', '@require_http_methods(["GET", "POST"])' + index = content_lines.index(function_define_line) + if !index.nil? + while index > 0 + index -= 1 + + preceding_definition = content_lines[index] + if preceding_definition.size > 0 && preceding_definition[0] == '@' + HTTP_METHOD_NAMES.each do |http_method_name| + method_name_match = preceding_definition.downcase.match /[^a-zA-Z0-9](#{http_method_name})[^a-zA-Z0-9]/ + if !method_name_match.nil? + suspicious_http_methods << http_method_name.upcase + end + end + end + + break + end + end + + lines.each do |line| + # Check if line has 'request.method == "GET"' similar pattern + if line.includes? "request.method" + suspicious_code = line.split("request.method")[1].strip + HTTP_METHOD_NAMES.each do |http_method_name| + method_name_match = suspicious_code.downcase.match /['"](#{http_method_name})['"]/ + if !method_name_match.nil? + suspicious_http_methods << http_method_name.upcase + end + end + end + + parse_params(line, suspicious_http_methods).each do |param| + suspicious_params << param + end + end + + suspicious_http_methods.uniq.each do |http_method_name| + endpoints << Endpoint.new(url, http_method_name, get_filtered_params(http_method_name, suspicious_params)) + end + + return endpoints + end + end + + # Class Based View + regex_http_method_names = HTTP_METHOD_NAMES.join "|" + class_start_index = content.index /class\s+#{function_or_class_name}\s*[\(:]/ + if !class_start_index.nil? + class_codeblock = parse_function_or_class(content[class_start_index..]) + if !class_codeblock.nil? + lines = class_codeblock.split "\n" + class_define_line = lines[0] + lines = lines[1..] + + # [TODO] Create a graph and use Django internal views + # Suspicious implicit class name for this class + # https://github.com/django/django/blob/main/django/views/generic/edit.py + if class_define_line.includes? "Form" + suspicious_http_methods << "GET" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Delete" + suspicious_http_methods << "DELETE" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Create" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Update" + suspicious_http_methods << "POST" + end + + # Check http methods (django.views.View) + lines.each do |line| + method_function_match = line.match(/\s+def\s+(#{regex_http_method_names})\s*\(/) + if !method_function_match.nil? + suspicious_http_methods << method_function_match[1].upcase + end + + parse_params(line, suspicious_http_methods).each do |param| + suspicious_params << param + end + end + + suspicious_http_methods.uniq.each do |http_method_name| + endpoints << Endpoint.new(url, http_method_name, get_filtered_params(http_method_name, suspicious_params)) + end + + return endpoints + end + end + + # GET is default http method + [Endpoint.new(url, "GET")] + end + + def parse_function_or_class(content : String) + lines = content.split("\n") + + indent_size = 0 + if lines.size > 0 + while indent_size < lines[0].size && lines[0][indent_size] == ' ' + # Only spaces, no tabs + indent_size += 1 + end + + indent_size += INDENT_SPACE_SIZE end - paths + if indent_size > 0 + double_quote_open, single_quote_open = [false] * 2 + double_comment_open, single_comment_open = [false] * 2 + end_index = lines[0].size + 1 + lines[1..].each do |line| + line_index = 0 + clear_line = line + while line_index < line.size + if line_index < line.size - 2 + if !single_quote_open && !double_quote_open + if !double_comment_open && line[line_index..line_index + 2] == "'''" + single_comment_open = !single_comment_open + line_index += 3 + next + elsif !single_comment_open && line[line_index..line_index + 2] == "\"\"\"" + double_comment_open = !double_comment_open + line_index += 3 + next + end + end + end + + if !single_comment_open && !double_comment_open + if !single_quote_open && line[line_index] == '"' && line[line_index - 1] != '\\' + double_quote_open = !double_quote_open + elsif !double_quote_open && line[line_index] == '\'' && line[line_index - 1] != '\\' + single_quote_open = !single_quote_open + elsif !single_quote_open && !double_quote_open && line[line_index] == '#' && line[line_index - 1] != '\\' + clear_line = line[..(line_index - 1)] + break + end + end + + # [TODO] Remove comments on codeblock + line_index += 1 + end + + open_status = single_comment_open || double_comment_open || single_quote_open || double_quote_open + if clear_line[0..(indent_size - 1)].strip == "" || open_status + end_index += line.size + 1 + else + break + end + end + + end_index -= 1 + return content[..end_index].strip + end + + nil end - def mapping_to_path(content : String, prefix : String = "") - paths = Array(String).new - content.scan(REGEX_URL_MAPPING) do |match| - next if match.size != 3 - path = match[1] - view = match[2] + def parse_params(line : String, endpoint_methods : Array(String)) + suspicious_params = Array(Param).new + + if line.includes? "request." + REQUEST_PARAM_FIELD_MAP.each do |field_name, tuple| + field_methods, noir_param_type = tuple + matches = line.scan(/request\.#{field_name}\[['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = line.scan(/request\.#{field_name}\.get\(['"]([^'"]*)['"]/) + end - path = path.gsub(/ /, "") - path = path.gsub(/^\^/, "") - path = path.gsub(/\$$/, "") + if matches.size != 0 + matches.each do |match| + next if match.size != 2 + param_name = match[1] + if field_name == "META" + if param_name.starts_with? "HTTP_" + param_name = param_name[5..] + end + elsif noir_param_type == "header" + if field_name == "COOKIES" + param_name = "Cookie['#{param_name}']" + end + end - filepath = nil - view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match| - next if include_pattern_match.size != 2 - filepath = "#{base_path}/#{include_pattern_match[1].gsub(".", "/")}.py" + # If it receives a specific parameter, it is considered to allow the method. + if !field_methods.nil? + field_methods.each do |field_method| + if !endpoint_methods.includes? field_method + endpoint_methods << field_method + end + end + end + + suspicious_params << Param.new(param_name, "", noir_param_type) + end + end + end + end + + if line.includes? "form.cleaned_data" + matches = line.scan(/form\.cleaned_data\[['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = line.scan(/form\.cleaned_data\.get\(['"]([^'"]*)['"]/) + end + + if matches.size != 0 + matches.each do |match| + next if match.size != 2 + suspicious_params << Param.new(match[1], "", "form") + end + end + end + + suspicious_params + end - if File.exists?(filepath) - new_django_urls = DjangoUrls.new("#{prefix}#{path}", filepath) - new_paths = get_paths(new_django_urls) - new_paths.each do |new_path| - paths << new_path + def get_filtered_params(method : String, params : Array(Param)) + filtered_params = Array(Param).new + upper_method = method.upcase + + params.each do |param| + is_support_param = false + support_methods = REQUEST_PARAM_TYPE_MAP.fetch(param.param_type, nil) + if !support_methods.nil? + support_methods.each do |support_method| + if upper_method == support_method.upcase + is_support_param = true + elsif support_method.upcase == "GET" && param.param_type == "query" + # The GET method allows parameters to be used in other methods as well. + is_support_param = true end end + else + is_support_param = true end - unless path.starts_with?("/") - path = "/#{path}" + filtered_params.each do |filtered_param| + if filtered_param.name == param.name && filtered_param.param_type == param.param_type + is_support_param = false + break + end + end + + if is_support_param + filtered_params << param end - paths << "#{prefix}#{path}" end - paths + filtered_params end end @@ -122,9 +556,12 @@ end struct DjangoUrls include JSON::Serializable - property prefix, filepath + property prefix, filepath, basepath - def initialize(@prefix : String, @filepath : String) + def initialize(@prefix : String, @filepath : String, @basepath : String) + if !File.directory? @basepath + raise "The basepath for DjangoUrls (#{@basepath}) does not exist or is not a directory." + end end end @@ -133,5 +570,8 @@ struct DjangoView property prefix, filepath, name def initialize(@prefix : String, @filepath : String, @name : String) + if !File.directory? @filepath + raise "The filepath for DjangoView (#{@filepath}) does not exist." + end end end