diff --git a/.gitignore b/.gitignore index 8886015edc..652dfc513a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ /config/memcached.yml /config/newrelic.yml /config/rails_env.rb +/config/sidekiq.yml /config/storage.yml /config/user_spam_scorer.yml /config/xapian.yml diff --git a/.ruby-style.yml b/.ruby-style.yml index 8c25fcf7c1..bef28240f3 100644 --- a/.ruby-style.yml +++ b/.ruby-style.yml @@ -37,6 +37,9 @@ AllCops: Bundler/DuplicatedGem: Enabled: false +Bundler/DuplicatedGroup: + Enabled: false + Bundler/GemComment: Enabled: false @@ -255,6 +258,7 @@ Layout/LineLength: - "^\\s*it\\s+.*do$" - "^\\s*context\\s+.*do$" - "^\\s*describe\\s+.*do$" + - "^RSpec\\.describe(\\s+|\\().*do$" - "^\\s*class\\s+[A-Z].*<.*" Exclude: - bin/setup @@ -579,6 +583,9 @@ Lint/MissingCopEnableDirective: Lint/MissingSuper: Enabled: false +Lint/MixedCaseRange: + Enabled: false + Lint/MixedRegexpCaptureTypes: Enabled: false @@ -645,6 +652,9 @@ Lint/RedundantCopEnableDirective: Lint/RedundantDirGlobSort: Enabled: false +Lint/RedundantRegexpQuantifiers: + Enabled: false + Lint/RedundantRequireStatement: Enabled: false @@ -968,6 +978,9 @@ Performance/IoReadlines: Performance/MapCompact: Enabled: false +Performance/MapMethodChain: + Enabled: false + Performance/MethodObjectAsBlock: Enabled: false @@ -1120,6 +1133,9 @@ Rails/ContentTag: Rails/CreateTableWithTimestamps: Enabled: false +Rails/DangerousColumnNames: + Enabled: false + Rails/Date: Enabled: false @@ -1285,6 +1301,9 @@ Rails/RakeEnvironment: Rails/ReadWriteAttribute: Enabled: false +Rails/RedundantActiveRecordAllMethod: + Enabled: false + Rails/RedundantAllowNil: Enabled: false @@ -1354,6 +1373,9 @@ Rails/SchemaComment: Rails/ScopeArgs: Enabled: false +Rails/SelectMap: + Enabled: false + Rails/ShortI18n: Enabled: false @@ -1402,6 +1424,9 @@ Rails/UnknownEnv: Rails/UnusedIgnoredColumns: Enabled: false +Rails/UnusedRenderContent: + Enabled: false + Rails/Validation: Enabled: false @@ -1643,6 +1668,9 @@ Style/EvalWithLocation: Style/EvenOdd: Enabled: true +Style/ExactRegexpMatch: + Enabled: false + Style/ExpandPathArguments: Enabled: false @@ -1953,6 +1981,9 @@ Style/RandomWithOffset: Style/RedundantArgument: Enabled: false +Style/RedundantArrayConstructor: + Enabled: false + Style/RedundantAssignment: Enabled: false @@ -1971,6 +2002,9 @@ Style/RedundantConditional: Style/RedundantConstantBase: Enabled: false +Style/RedundantCurrentDirectoryInPath: + Enabled: false + Style/RedundantDoubleSplatHashBraces: Enabled: false @@ -1986,6 +2020,9 @@ Style/RedundantFetchBlock: Style/RedundantFileExtensionInRequire: Enabled: false +Style/RedundantFilterChain: + Enabled: false + Style/RedundantFreeze: Enabled: true @@ -2007,9 +2044,15 @@ Style/RedundantParentheses: Style/RedundantPercentQ: Enabled: false +Style/RedundantRegexpArgument: + Enabled: false + Style/RedundantRegexpCharacterClass: Enabled: false +Style/RedundantRegexpConstructor: + Enabled: false + Style/RedundantRegexpEscape: Enabled: false @@ -2049,6 +2092,9 @@ Style/RescueStandardError: Style/ReturnNil: Enabled: false +Style/ReturnNilInPredicateMethodDefinition: + Enabled: false + Style/SafeNavigation: Enabled: false @@ -2077,6 +2123,9 @@ Style/SingleArgumentDig: Style/SingleLineBlockParams: Enabled: false +Style/SingleLineDoEndBlock: + Enabled: false + Style/SingleLineMethods: Enabled: true @@ -2206,6 +2255,9 @@ Style/WhileUntilModifier: Style/WordArray: Enabled: true +Style/YAMLFileRead: + Enabled: false + Style/YodaCondition: Enabled: false diff --git a/.vagrant.yml.example b/.vagrant.yml.example deleted file mode 100644 index 2e9751df20..0000000000 --- a/.vagrant.yml.example +++ /dev/null @@ -1,11 +0,0 @@ -fqdn: alaveteli.10.10.10.30.nip.io -ip: 10.10.10.30 -# Only use this on networks you trust -public_network: false -memory: 1536 -themes_dir: ../alaveteli-themes -os: bullseye64 -use_nfs: false -show_settings: false -# By default CPU count is calculated dynamically -cpus: 2 diff --git a/Gemfile b/Gemfile index 4f7a46777b..a12f23b495 100644 --- a/Gemfile +++ b/Gemfile @@ -79,24 +79,25 @@ # the new version. It is always preferable to upgrade our code. source 'https://rubygems.org' -gem 'rails', '~> 7.0.4' +gem 'rails', '~> 7.0.8' -gem 'pg', '~> 1.4.6' +gem 'pg', '~> 1.5.4' # New gem releases aren't being done. master is newer and supports Rails > 3.0 gem 'acts_as_versioned', git: 'https://github.com/mysociety/acts_as_versioned.git', ref: '13e928b' gem 'active_model_otp' -gem 'bcrypt', '~> 3.1.18' +gem 'bcrypt', '~> 3.1.19' gem 'cancancan', '~> 3.5.0' gem 'charlock_holmes', '~> 0.7.7' -gem 'dalli', '~> 3.2.4' +gem 'dalli', '~> 3.2.6' gem 'exception_notification', '~> 4.5.0' gem 'fancybox-rails', '~> 0.3.0' +gem 'friendly_id', '~> 5.5.0' gem 'gnuplot', '~> 2.6.0' gem 'htmlentities', '~> 4.3.0' -gem 'icalendar', '~> 2.8.0' -gem 'jquery-rails', '~> 4.5.1' +gem 'icalendar', '~> 2.9.0' +gem 'jquery-rails', '~> 4.6.0' gem 'jquery-ui-rails', '~> 6.0.0' gem 'json', '~> 2.6.2' gem 'holidays', '~> 8.6.0' @@ -104,19 +105,22 @@ gem 'iso_country_codes', '~> 0.7.8' gem 'mail', '~> 2.8.1' gem 'maxmind-db', '~> 1.0.0' gem 'mahoro', '~> 0.5' -gem 'nokogiri', '~> 1.14.3' +gem 'nokogiri', '~> 1.15.4' gem 'open4', '~> 1.3.0' -gem 'rack', '~> 2.2.6' -gem 'rack-utf8_sanitizer', '~> 1.8.0' -gem 'recaptcha', '~> 5.14.0', require: 'recaptcha/rails' +gem 'rack', '~> 2.2.8' +gem 'rack-utf8_sanitizer', '~> 1.9.1' +gem 'recaptcha', '~> 5.15.0', require: 'recaptcha/rails' gem 'matrix', '~> 0.4.2' gem 'mini_magick', '~> 4.12.0' +gem 'net-protocol', '~> 0.1.3' +gem 'redcarpet', '~> 3.6.0' gem 'redis', '~> 4.8.1' gem 'rolify', '~> 6.0.1' gem 'ruby-msg', '~> 1.5.0', git: 'https://github.com/mysociety/ruby-msg.git', branch: 'ascii-encoding' gem 'rubyzip', '~> 2.3.2' gem 'secure_headers', '~> 6.5.0' -gem 'sidekiq', '~> 6.5.8' +gem 'sidekiq', '~> 6.5.12' +gem 'sidekiq-limit_fetch', '~> 4.4.1' gem 'statistics2', '~> 0.54' gem 'strip_attributes', git: 'https://github.com/mysociety/strip_attributes.git', branch: 'globalize3-rails7' gem 'stripe', '~> 5.55.0' @@ -124,7 +128,7 @@ gem 'syck', '~> 1.4.1', require: false gem 'syslog_protocol', '~> 0.9.0' gem 'thin', '~> 1.8.2' gem 'vpim', '~> 13.11.11' -gem 'will_paginate', '~> 3.3.1' +gem 'will_paginate', '~> 4.0.0' gem 'xapian-full-alaveteli', '~> 1.4.22.1' gem 'xml-simple', '~> 1.1.9', require: 'xmlsimple' gem 'zip_tricks', '~> 5.6.0' @@ -133,11 +137,11 @@ gem 'zip_tricks', '~> 5.6.0' gem 'gender_detector', '~> 2.0.0' # Gems related to internationalisation -gem 'i18n', '~> 1.12.0' +gem 'i18n', '~> 1.14.1' gem 'rails-i18n', '~> 7.0.5' -gem 'gettext_i18n_rails', '~> 1.10.0' +gem 'gettext_i18n_rails', '~> 1.12.0' gem 'fast_gettext', '~> 2.3.0' -gem 'gettext', '~> 3.4.3' +gem 'gettext', '~> 3.4.7' gem 'globalize', '~> 6.2.1' gem 'locale', '~> 2.1.3' gem 'routing-filter', '~> 0.7.0' @@ -146,12 +150,13 @@ gem 'unidecoder', '~> 1.1.0' gem 'money', '~> 6.16.0' # mime-types 3.0.0 requires Ruby 2.0.0, and _something_ is trying to update it -gem 'mime-types', '< 3.0.0', require: false +gem 'mime-types', '< 4.0.0', require: false # Assets gem 'bootstrap-sass', '~> 2.3.2.2' -gem 'mini_racer', '~> 0.6.3' +gem 'mini_racer', '~> 0.8.0' gem 'sass-rails', '~> 5.0.8' +gem 'sprockets', git: 'https://github.com/rails/sprockets', ref: '3.x' gem 'uglifier', '~> 4.2.0' # Feature flags @@ -164,32 +169,32 @@ gem 'google-cloud-storage', '~> 1.44', require: false group :test do gem 'fivemat', '~> 1.3.7' - gem 'webmock', '~> 3.18.1' + gem 'webmock', '~> 3.19.1' gem 'simplecov', '~> 0.22.0' gem 'simplecov-lcov', '~> 0.7.0' - gem 'capybara', '~> 3.39.0' + gem 'capybara', '~> 3.39.2' gem 'stripe-ruby-mock', git: 'https://github.com/stripe-ruby-mock/stripe-ruby-mock', ref: '6ceea96' gem 'rails-controller-testing' end group :test, :development do - gem 'bullet', '~> 7.0.7' + gem 'bullet', '~> 7.1.2' gem 'factory_bot_rails', '~> 6.2.0' gem 'oink', '~> 0.10.1' gem 'rspec-activemodel-mocks', '~> 1.1.0' - gem 'rspec-rails', '~> 6.0.0' + gem 'rspec-rails', '~> 6.0.3' gem 'pry', '~> 0.14.2' end group :development do gem 'annotate', '< 3.2.1' - gem 'capistrano', '~> 2.15.0', '< 3.0.0' - gem 'net-ssh', '~> 7.1.0' + gem 'capistrano', '~> 2.15.11' + gem 'net-ssh', '~> 7.2.0' gem 'net-ssh-gateway', '>= 1.1.0', '< 3.0.0' gem 'launchy', '< 2.6.0' gem 'web-console', '>= 3.3.0' - gem 'rubocop', '~> 1.50.2', require: false + gem 'rubocop', '~> 1.57.1', require: false gem 'rubocop-performance', require: false gem 'rubocop-rails', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index b61e4f8c4a..80f214c23d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,6 +23,15 @@ GIT strip_attributes (1.12.0) activemodel (>= 3.0, < 8.0) +GIT + remote: https://github.com/rails/sprockets + revision: f4d3dae71ef29c44b75a49cfbf8032cce07b423a + ref: 3.x + specs: + sprockets (3.7.2) + concurrent-ruby (~> 1.0) + rack (> 1, < 3) + GIT remote: https://github.com/stripe-ruby-mock/stripe-ruby-mock revision: 6ceea9679bb573cb8bc6830f1bdf670b220a9859 @@ -39,101 +48,101 @@ PATH alaveteli_features (0.0.1) flipper (~> 0.10) flipper-active_record (~> 0.10) - mime-types (< 3.0.0) + mime-types (< 4.0.0) rails (~> 7.0.4) GEM remote: https://rubygems.org/ specs: - actioncable (7.0.4.3) - actionpack (= 7.0.4.3) - activesupport (= 7.0.4.3) + actioncable (7.0.8) + actionpack (= 7.0.8) + activesupport (= 7.0.8) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (7.0.4.3) - actionpack (= 7.0.4.3) - activejob (= 7.0.4.3) - activerecord (= 7.0.4.3) - activestorage (= 7.0.4.3) - activesupport (= 7.0.4.3) + actionmailbox (7.0.8) + actionpack (= 7.0.8) + activejob (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.0.4.3) - actionpack (= 7.0.4.3) - actionview (= 7.0.4.3) - activejob (= 7.0.4.3) - activesupport (= 7.0.4.3) + actionmailer (7.0.8) + actionpack (= 7.0.8) + actionview (= 7.0.8) + activejob (= 7.0.8) + activesupport (= 7.0.8) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.0) - actionpack (7.0.4.3) - actionview (= 7.0.4.3) - activesupport (= 7.0.4.3) - rack (~> 2.0, >= 2.2.0) + actionpack (7.0.8) + actionview (= 7.0.8) + activesupport (= 7.0.8) + rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (7.0.4.3) - actionpack (= 7.0.4.3) - activerecord (= 7.0.4.3) - activestorage (= 7.0.4.3) - activesupport (= 7.0.4.3) + actiontext (7.0.8) + actionpack (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.0.4.3) - activesupport (= 7.0.4.3) + actionview (7.0.8) + activesupport (= 7.0.8) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - active_model_otp (2.3.1) + active_model_otp (2.3.2) activemodel rotp (~> 6.2.0) - activejob (7.0.4.3) - activesupport (= 7.0.4.3) + activejob (7.0.8) + activesupport (= 7.0.8) globalid (>= 0.3.6) - activemodel (7.0.4.3) - activesupport (= 7.0.4.3) - activerecord (7.0.4.3) - activemodel (= 7.0.4.3) - activesupport (= 7.0.4.3) - activestorage (7.0.4.3) - actionpack (= 7.0.4.3) - activejob (= 7.0.4.3) - activerecord (= 7.0.4.3) - activesupport (= 7.0.4.3) + activemodel (7.0.8) + activesupport (= 7.0.8) + activerecord (7.0.8) + activemodel (= 7.0.8) + activesupport (= 7.0.8) + activestorage (7.0.8) + actionpack (= 7.0.8) + activejob (= 7.0.8) + activerecord (= 7.0.8) + activesupport (= 7.0.8) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (7.0.4.3) + activesupport (7.0.8) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - addressable (2.8.2) + addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) annotate (3.2.0) activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) ast (2.4.2) aws-eventstream (1.2.0) - aws-partitions (1.751.0) - aws-sdk-core (3.171.0) + aws-partitions (1.830.0) + aws-sdk-core (3.184.0) aws-eventstream (~> 1, >= 1.0.2) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.5) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.63.0) - aws-sdk-core (~> 3, >= 3.165.0) + aws-sdk-kms (1.72.0) + aws-sdk-core (~> 3, >= 3.184.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.121.0) - aws-sdk-core (~> 3, >= 3.165.0) + aws-sdk-s3 (1.136.0) + aws-sdk-core (~> 3, >= 3.181.0) aws-sdk-kms (~> 1) - aws-sigv4 (~> 1.4) - aws-sigv4 (1.5.2) + aws-sigv4 (~> 1.6) + aws-sigv4 (1.6.0) aws-eventstream (~> 1, >= 1.0.2) azure-core (0.1.15) faraday (~> 0.9) @@ -144,22 +153,23 @@ GEM faraday (~> 0.9) faraday_middleware (~> 0.10) nokogiri (~> 1.6, >= 1.6.8) - bcrypt (3.1.18) + base64 (0.1.1) + bcrypt (3.1.19) bindex (0.8.1) bootstrap-sass (2.3.2.2) sass (~> 3.2) builder (3.2.4) - bullet (7.0.7) + bullet (7.1.2) activesupport (>= 3.0.0) uniform_notifier (~> 1.11) cancancan (3.5.0) - capistrano (2.15.9) + capistrano (2.15.11) highline net-scp (>= 1.0.0) net-sftp (>= 2.0.0) net-ssh (>= 2.0.14) net-ssh-gateway (>= 1.1.0) - capybara (3.39.0) + capybara (3.39.2) addressable matrix mini_mime (>= 0.1.3) @@ -171,12 +181,12 @@ GEM charlock_holmes (0.7.7) coderay (1.1.3) concurrent-ruby (1.2.2) - connection_pool (2.3.0) + connection_pool (2.4.1) crack (0.4.5) rexml crass (1.0.6) daemons (1.4.1) - dalli (3.2.4) + dalli (3.2.6) dante (0.2.0) date (3.3.3) declarative (0.0.20) @@ -203,21 +213,25 @@ GEM faraday (>= 0.7.4, < 1.0) fast_gettext (2.3.0) fivemat (1.3.7) - flipper (0.24.1) - flipper-active_record (0.24.1) + flipper (0.28.3) + concurrent-ruby (< 2) + flipper-active_record (0.28.3) activerecord (>= 4.2, < 8) - flipper (~> 0.24.1) + flipper (~> 0.28.3) forwardable (1.3.3) + friendly_id (5.5.0) + activerecord (>= 4.0.0) gender_detector (2.0.0) - gettext (3.4.3) + gettext (3.4.7) erubi locale (>= 2.0.5) prime + racc text (>= 1.3.0) - gettext_i18n_rails (1.10.0) + gettext_i18n_rails (1.12.0) fast_gettext (>= 0.9.0) - globalid (1.1.0) - activesupport (>= 5.0) + globalid (1.2.1) + activesupport (>= 6.1) globalize (6.2.1) activemodel (>= 4.2, < 7.1) activerecord (>= 4.2, < 7.1) @@ -258,19 +272,19 @@ GEM os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) hashdiff (1.0.1) - highline (2.0.0) + highline (2.1.0) hodel_3000_compliant_logger (0.1.1) holidays (8.6.0) htmlentities (4.3.4) httpclient (2.8.3) - i18n (1.12.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) - icalendar (2.8.0) + icalendar (2.9.0) ice_cube (~> 0.16) ice_cube (0.16.4) iso_country_codes (0.7.8) jmespath (1.6.2) - jquery-rails (4.5.1) + jquery-rails (4.6.0) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) @@ -278,17 +292,18 @@ GEM railties (>= 3.2.16) json (2.6.3) jwt (2.6.0) + language_server-protocol (3.17.0.3) launchy (2.5.2) addressable (~> 2.8) - libv8-node (16.10.0.0) - libv8-node (16.10.0.0-aarch64-linux) - libv8-node (16.10.0.0-arm64-darwin) - libv8-node (16.10.0.0-x86_64-darwin) - libv8-node (16.10.0.0-x86_64-linux) + libv8-node (18.16.0.0) + libv8-node (18.16.0.0-aarch64-linux) + libv8-node (18.16.0.0-arm64-darwin) + libv8-node (18.16.0.0-x86_64-darwin) + libv8-node (18.16.0.0-x86_64-linux) locale (2.1.3) - loofah (2.19.1) + loofah (2.21.3) crass (~> 1.0.2) - nokogiri (>= 1.5.9) + nokogiri (>= 1.12.0) mahoro (0.5) mail (2.8.1) mini_mime (>= 0.1.1) @@ -300,44 +315,46 @@ GEM maxmind-db (1.0.0) memoist (0.16.2) method_source (1.0.0) - mime-types (2.99.3) + mime-types (3.5.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2023.0808) mini_magick (4.12.0) - mini_mime (1.1.2) - mini_portile2 (2.8.1) - mini_racer (0.6.3) - libv8-node (~> 16.10.0.0) - minitest (5.18.0) + mini_mime (1.1.5) + mini_portile2 (2.8.4) + mini_racer (0.8.0) + libv8-node (~> 18.16.0.0) + minitest (5.20.0) money (6.16.0) i18n (>= 0.6.4, <= 2) multi_json (1.15.0) multipart-post (2.2.3) - net-imap (0.3.4) + net-imap (0.3.7) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.1.3) timeout - net-scp (1.2.1) - net-ssh (>= 2.6.5) - net-sftp (2.1.2) - net-ssh (>= 2.6.5) + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) + net-sftp (4.0.0) + net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.3.3) net-protocol - net-ssh (7.1.0) + net-ssh (7.2.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) - nio4r (2.5.8) - nokogiri (1.14.3) - mini_portile2 (~> 2.8.0) + nio4r (2.5.9) + nokogiri (1.15.4) + mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.14.3-aarch64-linux) + nokogiri (1.15.4-aarch64-linux) racc (~> 1.4) - nokogiri (1.14.3-arm64-darwin) + nokogiri (1.15.4-arm64-darwin) racc (~> 1.4) - nokogiri (1.14.3-x86_64-darwin) + nokogiri (1.15.4-x86_64-darwin) racc (~> 1.4) - nokogiri (1.14.3-x86_64-linux) + nokogiri (1.15.4-x86_64-linux) racc (~> 1.4) oink (0.10.1) activerecord @@ -345,60 +362,64 @@ GEM open4 (1.3.4) os (1.1.4) parallel (1.23.0) - parser (3.2.2.0) + parser (3.2.2.4) ast (~> 2.4.1) - pg (1.4.6) + racc + pg (1.5.4) prime (0.1.2) forwardable singleton pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.1) - racc (1.6.2) - rack (2.2.6.4) + public_suffix (5.0.3) + racc (1.7.1) + rack (2.2.8) rack-test (2.1.0) rack (>= 1.3) - rack-utf8_sanitizer (1.8.0) + rack-utf8_sanitizer (1.9.1) rack (>= 1.0, < 4.0) - rails (7.0.4.3) - actioncable (= 7.0.4.3) - actionmailbox (= 7.0.4.3) - actionmailer (= 7.0.4.3) - actionpack (= 7.0.4.3) - actiontext (= 7.0.4.3) - actionview (= 7.0.4.3) - activejob (= 7.0.4.3) - activemodel (= 7.0.4.3) - activerecord (= 7.0.4.3) - activestorage (= 7.0.4.3) - activesupport (= 7.0.4.3) + rails (7.0.8) + actioncable (= 7.0.8) + actionmailbox (= 7.0.8) + actionmailer (= 7.0.8) + actionpack (= 7.0.8) + actiontext (= 7.0.8) + actionview (= 7.0.8) + activejob (= 7.0.8) + activemodel (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) bundler (>= 1.15.0) - railties (= 7.0.4.3) + railties (= 7.0.8) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) activesupport (>= 5.0.1.rc1) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.5.0) - loofah (~> 2.19, >= 2.19.1) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) rails-i18n (7.0.5) i18n (>= 0.7, < 2) railties (>= 6.0.0, < 8) - railties (7.0.4.3) - actionpack (= 7.0.4.3) - activesupport (= 7.0.4.3) + railties (7.0.8) + actionpack (= 7.0.8) + activesupport (= 7.0.8) method_source rake (>= 12.2) thor (~> 1.0) zeitwerk (~> 2.5) rainbow (3.1.1) rake (13.0.6) - recaptcha (5.14.0) + recaptcha (5.15.0) + redcarpet (3.6.0) redis (4.8.1) - regexp_parser (2.8.0) + regexp_parser (2.8.2) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -406,9 +427,9 @@ GEM request_store (1.5.1) rack (>= 1.4) retriable (3.1.2) - rexml (3.2.5) + rexml (3.2.6) rolify (6.0.1) - rotp (6.2.0) + rotp (6.2.2) routing-filter (0.7.0) actionpack (>= 6.1) activesupport (>= 6.1) @@ -416,39 +437,41 @@ GEM activemodel (>= 3.0) activesupport (>= 3.0) rspec-mocks (>= 2.99, < 4.0) - rspec-core (3.11.0) - rspec-support (~> 3.11.0) - rspec-expectations (3.11.1) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-mocks (3.11.1) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-rails (6.0.1) + rspec-support (~> 3.12.0) + rspec-rails (6.0.3) actionpack (>= 6.1) activesupport (>= 6.1) railties (>= 6.1) - rspec-core (~> 3.11) - rspec-expectations (~> 3.11) - rspec-mocks (~> 3.11) - rspec-support (~> 3.11) - rspec-support (3.11.1) - rubocop (1.50.2) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rspec-support (~> 3.12) + rspec-support (3.12.0) + rubocop (1.57.1) + base64 (~> 0.1.1) json (~> 2.3) + language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.0.0) + parser (>= 3.2.2.4) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.0, < 2.0) + rubocop-ast (>= 1.28.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.28.0) + rubocop-ast (1.29.0) parser (>= 3.2.1.0) - rubocop-performance (1.17.1) + rubocop-performance (1.19.1) rubocop (>= 1.7.0, < 2.0) rubocop-ast (>= 0.4.0) - rubocop-rails (2.19.1) + rubocop-rails (2.21.2) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) @@ -463,10 +486,12 @@ GEM sprockets-rails (>= 2.0, < 4.0) tilt (>= 1.1, < 3) secure_headers (6.5.0) - sidekiq (6.5.8) + sidekiq (6.5.12) connection_pool (>= 2.2.5, < 3) rack (~> 2.0) redis (>= 4.5.0, < 5) + sidekiq-limit_fetch (4.4.1) + sidekiq (>= 6) signet (0.17.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) @@ -480,9 +505,6 @@ GEM simplecov-lcov (0.7.0) simplecov_json_formatter (0.1.4) singleton (0.1.1) - sprockets (3.7.2) - concurrent-ruby (~> 1.0) - rack (> 1, < 3) sprockets-rails (3.4.2) actionpack (>= 5.2) activesupport (>= 5.2) @@ -496,9 +518,9 @@ GEM daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) rack (>= 1, < 3) - thor (1.2.1) + thor (1.2.2) tilt (2.0.10) - timeout (0.3.2) + timeout (0.4.0) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) @@ -506,24 +528,24 @@ GEM uglifier (4.2.0) execjs (>= 0.3.0, < 3) unicode (0.4.4.4) - unicode-display_width (2.4.2) + unicode-display_width (2.5.0) unidecoder (1.1.2) uniform_notifier (1.16.0) vpim (13.11.11) - web-console (4.2.0) + web-console (4.2.1) actionview (>= 6.0.0) activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) - webmock (3.18.1) + webmock (3.19.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.7.0) - websocket-driver (0.7.5) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - will_paginate (3.3.1) + will_paginate (4.0.0) xapian-full-alaveteli (1.4.22.1) mini_portile2 (~> 2.8) rake (~> 13.0) @@ -531,7 +553,7 @@ GEM rexml xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.7) + zeitwerk (2.6.11) zip_tricks (5.6.0) PLATFORMS @@ -548,31 +570,32 @@ DEPENDENCIES annotate (< 3.2.1) aws-sdk-s3 azure-storage - bcrypt (~> 3.1.18) + bcrypt (~> 3.1.19) bootstrap-sass (~> 2.3.2.2) - bullet (~> 7.0.7) + bullet (~> 7.1.2) cancancan (~> 3.5.0) - capistrano (~> 2.15.0, < 3.0.0) - capybara (~> 3.39.0) + capistrano (~> 2.15.11) + capybara (~> 3.39.2) charlock_holmes (~> 0.7.7) - dalli (~> 3.2.4) + dalli (~> 3.2.6) exception_notification (~> 4.5.0) factory_bot_rails (~> 6.2.0) fancybox-rails (~> 0.3.0) fast_gettext (~> 2.3.0) fivemat (~> 1.3.7) + friendly_id (~> 5.5.0) gender_detector (~> 2.0.0) - gettext (~> 3.4.3) - gettext_i18n_rails (~> 1.10.0) + gettext (~> 3.4.7) + gettext_i18n_rails (~> 1.12.0) globalize (~> 6.2.1) gnuplot (~> 2.6.0) google-cloud-storage (~> 1.44) holidays (~> 8.6.0) htmlentities (~> 4.3.0) - i18n (~> 1.12.0) - icalendar (~> 2.8.0) + i18n (~> 1.14.1) + icalendar (~> 2.9.0) iso_country_codes (~> 0.7.8) - jquery-rails (~> 4.5.1) + jquery-rails (~> 4.6.0) jquery-ui-rails (~> 6.0.0) json (~> 2.6.2) launchy (< 2.6.0) @@ -581,38 +604,42 @@ DEPENDENCIES mail (~> 2.8.1) matrix (~> 0.4.2) maxmind-db (~> 1.0.0) - mime-types (< 3.0.0) + mime-types (< 4.0.0) mini_magick (~> 4.12.0) - mini_racer (~> 0.6.3) + mini_racer (~> 0.8.0) money (~> 6.16.0) - net-ssh (~> 7.1.0) + net-protocol (~> 0.1.3) + net-ssh (~> 7.2.0) net-ssh-gateway (>= 1.1.0, < 3.0.0) - nokogiri (~> 1.14.3) + nokogiri (~> 1.15.4) oink (~> 0.10.1) open4 (~> 1.3.0) - pg (~> 1.4.6) + pg (~> 1.5.4) pry (~> 0.14.2) - rack (~> 2.2.6) - rack-utf8_sanitizer (~> 1.8.0) - rails (~> 7.0.4) + rack (~> 2.2.8) + rack-utf8_sanitizer (~> 1.9.1) + rails (~> 7.0.8) rails-controller-testing rails-i18n (~> 7.0.5) - recaptcha (~> 5.14.0) + recaptcha (~> 5.15.0) + redcarpet (~> 3.6.0) redis (~> 4.8.1) rolify (~> 6.0.1) routing-filter (~> 0.7.0) rspec-activemodel-mocks (~> 1.1.0) - rspec-rails (~> 6.0.0) - rubocop (~> 1.50.2) + rspec-rails (~> 6.0.3) + rubocop (~> 1.57.1) rubocop-performance rubocop-rails ruby-msg (~> 1.5.0)! rubyzip (~> 2.3.2) sass-rails (~> 5.0.8) secure_headers (~> 6.5.0) - sidekiq (~> 6.5.8) + sidekiq (~> 6.5.12) + sidekiq-limit_fetch (~> 4.4.1) simplecov (~> 0.22.0) simplecov-lcov (~> 0.7.0) + sprockets! statistics2 (~> 0.54) strip_attributes! stripe (~> 5.55.0) @@ -625,8 +652,8 @@ DEPENDENCIES unidecoder (~> 1.1.0) vpim (~> 13.11.11) web-console (>= 3.3.0) - webmock (~> 3.18.1) - will_paginate (~> 3.3.1) + webmock (~> 3.19.1) + will_paginate (~> 4.0.0) xapian-full-alaveteli (~> 1.4.22.1) xml-simple (~> 1.1.9) zip_tricks (~> 5.6.0) diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index f08833608d..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,243 +0,0 @@ -require 'pp' -require 'yaml' -# Welcome! Thanks for taking an interest in contributing to Alaveteli. -# This Vagrantfile should get you started with the minimum of fuss. -# -# Usage -# ===== -# -# Install a vagrant plugin (if you don't already have it) that -# automatically install guest additions -# -# # Host -# $ vagrant plugin install vagrant-vbguest -# $ vagrant vbguest -# -# Get a copy of Alaveteli from GitHub and create the Vagrant instance -# -# # Host -# $ git clone git@github.com:mysociety/alaveteli.git -# $ cd alaveteli -# $ git submodule update --init -# $ vagrant --no-color up -# -# You should now be able to ssh in to the guest and run the test suite -# -# # Host -# $ vagrant ssh -# -# # Guest -# $ cd /home/vagrant/alaveteli -# $ bundle exec rake spec -# -# Run the rails server and visit the application in your host browser -# at http://10.10.10.30:3000 -# -# # Guest -# bundle exec rails server -b 0.0.0.0 -# -# -# Log-in to the Vagrant instance -# ================================ -# -# Once the application is running, you can login with any of the sample users -# that get created automatically. You can find more details of these users in -# spec/fixtures/users.yml. -# -# Customizing the Vagrant instance -# ================================ -# -# This Vagrantfile allows customisation of some aspects of the virtaual machine -# See the customization options below for details. -# -# The options can be set either by prefixing the vagrant command, using -# `.vagrant.yml`, or by exporting to the environment. -# -# # Prefixing the command -# $ ALAVETELI_VAGRANT_MEMORY=2048 vagrant up -# -# # .vagrant.yml -# $ echo "memory: 2048" >> .vagrant.yml -# $ vagrant up -# -# # Exporting to the environment -# $ export ALAVETELI_VAGRANT_MEMORY=2048 -# $ vagrant up -# -# All have the same effect, but exporting will retain the variable for the -# duration of your shell session, whereas `.vagrant.yml` will be persistent. -# The environment takes precedence over `.vagrant.yml`. -# -# Using Themes -# ------------ -# -# You can also use the built in theme switcher (script/switch-theme.rb). The -# ALAVETELI_THEMES_DIR will be shared in to /home/vagrant/alaveteli-themes so -# that the default location is used on the guest. You can use the env var -# ALAVETELI_THEMES_DIR to change where this Vagrantfile looks for the themes -# directory on the host. - -def cpu_count - host = RbConfig::CONFIG['host_os'] - # Give VM access to all cpu cores on the host - if host =~ /darwin/ - `sysctl -n hw.ncpu`.to_i - elsif host =~ /linux/ - `nproc`.to_i - else # sorry Windows folks, I can't help you - 1 - end -end - -# Customization Options -# ===================== -# -# Defaults can be overridden either in `.vagrant.yml` with the same key name, or -# via the environment by prefixing the key with `ALAVETELI_VAGRANT_` and -# upcasing. Boolean values can be set to `false` in the environment with "0", -# "false" or "no". -DEFAULTS = { - 'fqdn' => 'alaveteli.10.10.10.30.nip.io', - 'ip' => '10.10.10.30', - 'public_network' => false, - 'memory' => 1536, - 'themes_dir' => '../alaveteli-themes', - 'os' => 'bullseye64', - 'name' => 'default', - 'use_nfs' => false, - 'show_settings' => false, - 'cpus' => cpu_count -}.freeze - -env = DEFAULTS.keys.reduce({}) do |memo, key| - value = ENV["ALAVETELI_VAGRANT_#{ key.upcase }"] - value = false if %w(0 false no).include?(value) - memo[key] = value unless value.nil? - memo -end - -settings_file_path = File.dirname(__FILE__) + '/.vagrant.yml' -settings_file = if File.exist?(settings_file_path) - YAML.load(File.read(settings_file_path)) -else - {} -end - -SUPPORTED_OPERATING_SYSTEMS = { - 'focal64' => { - box: 'ubuntu/focal64', - box_url: 'https://app.vagrantup.com/ubuntu/boxes/focal64' - }, - 'bullseye64' => { - box: 'debian/bullseye64', - box_url: 'https://app.vagrantup.com/debian/boxes/bullseye64' - } -} - -def os - SUPPORTED_OPERATING_SYSTEMS.fetch(SETTINGS['os'], box: SETTINGS['os']) -end - -SETTINGS = DEFAULTS.merge(settings_file).merge(env).freeze - -if SETTINGS['show_settings'] - puts 'Current machine settings:' - puts "\n" - pp SETTINGS - puts "\n" - puts 'Current OS settings:' - puts "\n" - pp os - puts "\n" -end - -VAGRANTFILE_API_VERSION = '2' - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = os[:box] - config.vm.define SETTINGS['name'] - config.vm.box_url = os[:box_url] - config.vm.hostname = "alaveteli-#{ SETTINGS['os'] }" - - config.vm.network :public_network if SETTINGS['public_network'] - - config.vm.network :private_network, ip: SETTINGS['ip'] - - config.vm.synced_folder '.', '/vagrant', disabled: true - - if SETTINGS['use_nfs'] - config.vm.synced_folder '.', '/home/vagrant/alaveteli', nfs: true - else - config.vm.synced_folder '.', - '/home/vagrant/alaveteli', - owner: 'vagrant', - group: 'vagrant' - end - - if File.directory?(SETTINGS['themes_dir']) - if SETTINGS['use_nfs'] - config.vm.synced_folder SETTINGS['themes_dir'], - '/home/vagrant/alaveteli-themes', - nfs: true - else - config.vm.synced_folder SETTINGS['themes_dir'], - '/home/vagrant/alaveteli-themes', - owner: 'vagrant', - group: 'vagrant' - end - end - - config.ssh.forward_agent = true - - # The bundle install fails unless you have quite a large amount of - # memory; insist on 1.5GiB: - config.vm.provider 'virtualbox' do |vb| - vb.customize ['modifyvm', :id, '--memory', SETTINGS['memory']] - vb.customize ['modifyvm', :id, '--cpus', SETTINGS['cpus']] - end - - config.vm.provision :shell, keep_color: true, inline: <<-EOF - if [[ -f "/home/vagrant/alaveteli/commonlib/bin/install-site.sh" ]] - then - /home/vagrant/alaveteli/commonlib/bin/install-site.sh \ - --dev \ - alaveteli \ - vagrant \ - #{ SETTINGS['fqdn'] } - else - echo "Couldn't find provisioning script." >&2 - echo "Did you forget to run git submodule update --init?" >&2 - exit 1 - fi -EOF - - # Append basic usage instructions to the MOTD - motd = <<-EOF -To start your alaveteli instance: -* cd alaveteli -* bundle exec rails server -b 0.0.0.0 -EOF - - config.vm.provision :shell, keep_color: true, inline: "echo '#{ motd }' >> /etc/motd.tail" - - # Display next steps info at the end of a successful install - instructions = <<-EOF - -Welcome to your new Alaveteli development site! - -If you are planning to use a custom theme, you should create -an `alaveteli-themes` folder at the same level as your `alaveteli` -code folder to hold your theme repositories so that your -Vagrant box will see your theme folders when using the -switch-theme.rb script (take a look at the documentation in -the script/switch-theme.rb file for more information). - -Full instructions for customising your install can be found online: -http://alaveteli.org/docs/customising/ - -Type `vagrant ssh` to log into the Vagrant box to start the site -or run the test suite -EOF - - config.vm.provision :shell, keep_color: true, inline: "echo '#{ instructions }'" -end diff --git a/app/assets/stylesheets/admin.scss b/app/assets/stylesheets/admin.scss index 71379d7ca0..5ce1bb4c09 100644 --- a/app/assets/stylesheets/admin.scss +++ b/app/assets/stylesheets/admin.scss @@ -282,6 +282,17 @@ body.admin { padding: 2em; } +/* Toolbox */ +.toolbox { + summary { + margin-bottom: 1em; + } + + .toolbox-tool { + margin-bottom: 1em; + } +} + /* Bootstrap Extensions */ /* These must come last because we @import bootstrap in .admin */ diff --git a/app/assets/stylesheets/admin/_print_style.scss b/app/assets/stylesheets/admin/_print_style.scss index f5f9c75bf8..8919efa815 100644 --- a/app/assets/stylesheets/admin/_print_style.scss +++ b/app/assets/stylesheets/admin/_print_style.scss @@ -76,7 +76,7 @@ tfoot { // This will avoid fieldset element to have a page-break. // The fieldset have a display: none on line 21m I'm leaving this code in case -// we decide to make the elemnets visible again. +// we decide to make the elements visible again. fieldset { page-break-inside: avoid; } diff --git a/app/assets/stylesheets/responsive/_new_request_layout.scss b/app/assets/stylesheets/responsive/_new_request_layout.scss index 95c840afed..7ec77084c6 100644 --- a/app/assets/stylesheets/responsive/_new_request_layout.scss +++ b/app/assets/stylesheets/responsive/_new_request_layout.scss @@ -102,7 +102,7 @@ padding-left: 20px; input[type="radio"] { - // Margin-left: Prevents double line questions to have an uneven vertical aligment. + // Margin-left: Prevents double line questions to have an uneven vertical alignment. // Margin-bottom: Fixes the large gap between a question with two lines. margin: 0 3px 0 -20px; } diff --git a/app/assets/stylesheets/responsive/_request_layout.scss b/app/assets/stylesheets/responsive/_request_layout.scss index 40f22b3511..1be8722a07 100644 --- a/app/assets/stylesheets/responsive/_request_layout.scss +++ b/app/assets/stylesheets/responsive/_request_layout.scss @@ -119,6 +119,18 @@ } } +.request-header__closed_to_correspondence , +.request-header__restricted_correspondence { + display: block; + margin-top: 2em; +} + +#request_upload_response .request-header__closed_to_correspondence { + @include grid-column(12); + margin-top: 0.5em; + margin-bottom: 1em; +} + .feed_link_toolbar { width: 100%; diff --git a/app/assets/stylesheets/responsive/_request_style.scss b/app/assets/stylesheets/responsive/_request_style.scss index 7ccc08efb8..ced2d23e93 100644 --- a/app/assets/stylesheets/responsive/_request_style.scss +++ b/app/assets/stylesheets/responsive/_request_style.scss @@ -34,6 +34,25 @@ $color-delivery-unknown: darken(#F3BD2A, 20%) !default; // yellow } } +#request_upload_response .request-header__closed_to_correspondence { + font-size: 2em; + font-weight: bold; + line-height: 1em; + color: #333; + + a { + color: #2688dc; + &:hover, + &:active, + &:focus { + color: #333333; + } + &:visited { + color: darken(#2688dc, 10%); + } + } +} + a.track-request-action { white-space: nowrap; text-align: left; diff --git a/app/controllers/admin/changelog_controller.rb b/app/controllers/admin/changelog_controller.rb new file mode 100644 index 0000000000..6a897f1f7e --- /dev/null +++ b/app/controllers/admin/changelog_controller.rb @@ -0,0 +1,9 @@ +# Controller to render the changelog notes in a more human-friendly way within +# the admin interface. +class Admin::ChangelogController < AdminController + def index + markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML.new) + text = File.read(Rails.root + 'doc/CHANGES.md') + @changelog = markdown.render(text).html_safe + end +end diff --git a/app/controllers/admin_controller.rb b/app/controllers/admin_controller.rb index ba1eeab5c4..48ea109850 100644 --- a/app/controllers/admin_controller.rb +++ b/app/controllers/admin_controller.rb @@ -1,5 +1,5 @@ # controllers/admin.rb: -# All admin controllers are dervied from this. +# All admin controllers are derived from this. # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. # Email: hello@mysociety.org; WWW: http://www.mysociety.org/ diff --git a/app/controllers/admin_general_controller.rb b/app/controllers/admin_general_controller.rb index ba9a506f79..8d125e732c 100644 --- a/app/controllers/admin_general_controller.rb +++ b/app/controllers/admin_general_controller.rb @@ -5,6 +5,7 @@ # Email: hello@mysociety.org; WWW: http://www.mysociety.org/ class AdminGeneralController < AdminController + include AdminGeneralTimelineHelper def index # Tasks to do @@ -170,75 +171,40 @@ def debug private - def get_date_back_to_utc - date_back_to = if params[:hour] - Time.zone.now - 1.hour - elsif params[:day] - Time.zone.now - 1.day - elsif params[:week] - Time.zone.now - 1.week - elsif params[:month] - Time.zone.now - 1.month - elsif params[:all] - Time.zone.now - 1000.years - else - Time.zone.now - 2.days - end - date_back_to.getutc - end - - def get_event_type - if params[:event_type] == 'authority_change' - 'Authority changes' - elsif params[:event_type] == 'info_request_event' - 'Request events' - else - "Events" - end - end - def get_events_title - title = if params[:hour] - "#{get_event_type} in last hour" - elsif params[:day] - "#{get_event_type} in last day" - elsif params[:week] - "#{get_event_type} in last week" - elsif params[:month] - "#{get_event_type} in last month" - elsif params[:all] - "#{get_event_type}, all time" + if current_time_filter == 'All time' + "#{current_event_type}, all time" else - "#{get_event_type} in last two days" + "#{current_event_type} in the last #{current_time_filter.downcase}" end end def get_timestamps # Get an array of event attributes within the timespan in the format # [id, type_of_model, event_timestamp] - # Note that the relevent date for InfoRequestEvents is creation, but - # for PublicBodyVersions is update thoughout + # Note that the relevant date for InfoRequestEvents is creation, but + # for PublicBodyVersions is update throughout connection = InfoRequestEvent.connection - authority_change_clause = "SELECT id, 'PublicBodyVersion', - updated_at AS timestamp - FROM #{PublicBody.versioned_class.table_name} - WHERE updated_at > '#{get_date_back_to_utc}'" - - info_request_event_clause = "SELECT id, 'InfoRequestEvent', - created_at AS timestamp - FROM info_request_events - WHERE created_at > '#{get_date_back_to_utc}'" - - timestamps = if params[:event_type] == 'authority_change' - connection.select_rows("#{authority_change_clause} - ORDER by timestamp desc") - elsif params[:event_type] == 'info_request_event' - connection.select_rows("#{info_request_event_clause} - ORDER by timestamp desc") + + authority_change_scope = PublicBody.versioned_class. + select("id, 'PublicBodyVersion', updated_at AS timestamp"). + where(updated_at: start_date...). + order(timestamp: :desc) + + info_request_event_scope = InfoRequestEvent. + select("id, 'InfoRequestEvent', created_at AS timestamp"). + where(created_at: start_date...). + order(timestamp: :desc) + + case params[:event_type] + when 'authority_change' + connection.select_rows(authority_change_scope.to_sql) + when 'info_request_event' + connection.select_rows(info_request_event_scope.to_sql) else - connection.select_rows("#{info_request_event_clause} + connection.select_rows("#{info_request_event_scope.unscope(:order).to_sql} UNION - #{authority_change_clause} + #{authority_change_scope.unscope(:order).to_sql} ORDER by timestamp desc") end end diff --git a/app/controllers/admin_incoming_message_controller.rb b/app/controllers/admin_incoming_message_controller.rb index e70d3f9990..d6a0443105 100644 --- a/app/controllers/admin_incoming_message_controller.rb +++ b/app/controllers/admin_incoming_message_controller.rb @@ -78,30 +78,34 @@ def bulk_destroy end def redeliver - - message_ids = params[:url_title].split(",").each(&:strip) + message_ids = params[:url_title].split(',').map(&:strip) previous_request = @incoming_message.info_request destination_request = nil if message_ids.empty? - flash[:error] = "You must supply at least one request to redeliver the message to." + flash[:error] = + 'You must supply at least one request to redeliver the message to.' + return redirect_to admin_request_url(previous_request) end ActiveRecord::Base.transaction do message_ids.each do |m| - if m.match(/^[0-9]+$/) - destination_request = InfoRequest.find_by_id(m.to_i) - else - destination_request = InfoRequest.find_by_url_title!(m) - end + destination_request = + if m.match(/^[0-9]+$/) + InfoRequest.find_by_id(m.to_i) + else + InfoRequest.find_by_url_title!(m) + end + if destination_request.nil? - flash[:error] = "Failed to find destination request '" + m + "'" + flash[:error] = "Failed to find destination request '#{m}'" return redirect_to admin_request_url(previous_request) end raw_email_data = @incoming_message.raw_email.data mail = MailHandler.mail_from_raw_email(raw_email_data) + destination_request. receive(mail, raw_email_data, @@ -114,12 +118,15 @@ def redeliver deleted_incoming_message_id: @incoming_message.id ) - flash[:notice] = "Message has been moved to request(s). Showing the last one:" + flash[:notice] = + 'Message has been moved to request(s). Showing the last one:' end + # expire cached files previous_request.expire @incoming_message.destroy end + redirect_to admin_request_url(destination_request) end diff --git a/app/controllers/admin_outgoing_message_controller.rb b/app/controllers/admin_outgoing_message_controller.rb index 370d5c31fe..f057c715c9 100644 --- a/app/controllers/admin_outgoing_message_controller.rb +++ b/app/controllers/admin_outgoing_message_controller.rb @@ -85,7 +85,7 @@ def resend def outgoing_message_params if params[:outgoing_message] params.require(:outgoing_message). - permit(:prominence, :prominence_reason, :body, :tag_string) + permit(:prominence, :prominence_reason, :body, :tag_string, :from_name) else {} end diff --git a/app/controllers/admin_public_body_controller.rb b/app/controllers/admin_public_body_controller.rb index ff59aec23c..494edd9eb7 100644 --- a/app/controllers/admin_public_body_controller.rb +++ b/app/controllers/admin_public_body_controller.rb @@ -269,8 +269,8 @@ def public_body_params translatable_params( params[:public_body], translated_keys: [:locale, :name, :short_name, :request_email, - :publication_scheme], - general_keys: [:tag_string, :home_page, :disclosure_log, + :publication_scheme, :disclosure_log], + general_keys: [:tag_string, :home_page, :last_edit_comment, :last_edit_editor] ) end diff --git a/app/controllers/admin_user_slug_controller.rb b/app/controllers/admin_user_slug_controller.rb new file mode 100644 index 0000000000..2b4e52b9e7 --- /dev/null +++ b/app/controllers/admin_user_slug_controller.rb @@ -0,0 +1,22 @@ +## +# Controller responsible for remove user slugs +# +class AdminUserSlugController < AdminController + before_action :set_admin_user, :set_slug + + def destroy + @slug.destroy unless @slug.slug == @admin_user.url_name + redirect_to [:admin, @admin_user] + end + + private + + def set_admin_user + # Don't use @user as that is any logged in user + @admin_user = User.find(params[:user_id]) + end + + def set_slug + @slug = @admin_user.slugs.find(params[:id]) + end +end diff --git a/app/controllers/admin_users_account_anonymising_controller.rb b/app/controllers/admin_users_account_anonymising_controller.rb new file mode 100644 index 0000000000..5e926345ec --- /dev/null +++ b/app/controllers/admin_users_account_anonymising_controller.rb @@ -0,0 +1,26 @@ +## +# Controller for anonymising user accounts +# +class AdminUsersAccountAnonymisingController < AdminController + before_action :set_anonymised_user + + def create + if anonymise + flash[:notice] = 'The user was anonymised.' + else + flash[:error] = 'Something went wrong. The user could not be anonymised.' + end + + redirect_to admin_user_path(@anonymised_user) + end + + private + + def set_anonymised_user + @anonymised_user = User.find(params[:user_id]) + end + + def anonymise + @anonymised_user.anonymise! + end +end diff --git a/app/controllers/admin_users_account_closing_controller.rb b/app/controllers/admin_users_account_closing_controller.rb new file mode 100644 index 0000000000..75e26fef7c --- /dev/null +++ b/app/controllers/admin_users_account_closing_controller.rb @@ -0,0 +1,27 @@ +## +# Controller for closing user accounts +# +class AdminUsersAccountClosingController < AdminController + before_action :set_closed_user + + def create + if close + flash[:notice] = 'The user account was closed.' + else + flash[:error] = + 'Something went wrong. The user account could not be closed.' + end + + redirect_to admin_user_path(@closed_user) + end + + private + + def set_closed_user + @closed_user = User.find(params[:user_id]) + end + + def close + @closed_user.close + end +end diff --git a/app/controllers/admin_users_account_erasing_controller.rb b/app/controllers/admin_users_account_erasing_controller.rb new file mode 100644 index 0000000000..fea6636b5d --- /dev/null +++ b/app/controllers/admin_users_account_erasing_controller.rb @@ -0,0 +1,26 @@ +## +# Controller for erasing accounts +# +class AdminUsersAccountErasingController < AdminController + before_action :set_erased_user + + def create + if erase + flash[:notice] = 'The user was erased.' + else + flash[:error] = 'Something went wrong. The user could not be erased.' + end + + redirect_to admin_user_path(@erased_user) + end + + private + + def set_erased_user + @erased_user = User.find(params[:user_id]) + end + + def erase + @erased_user.erase + end +end diff --git a/app/controllers/admin_users_account_suspensions_controller.rb b/app/controllers/admin_users_account_suspensions_controller.rb index 4db9c1a91e..137f83e05b 100644 --- a/app/controllers/admin_users_account_suspensions_controller.rb +++ b/app/controllers/admin_users_account_suspensions_controller.rb @@ -24,11 +24,7 @@ def set_suspended_user end def suspend - if params[:close_and_anonymise] - @suspended_user.close_and_anonymise - else - @suspended_user.update(ban_text: @suspension_reason) - end + @suspended_user.update(ban_text: @suspension_reason) end def set_suspension_reason diff --git a/app/controllers/alaveteli_pro/invoices_controller.rb b/app/controllers/alaveteli_pro/invoices_controller.rb new file mode 100644 index 0000000000..03c297623b --- /dev/null +++ b/app/controllers/alaveteli_pro/invoices_controller.rb @@ -0,0 +1,8 @@ +## +# Controller to allow Pro user to access past invoices. +# +class AlaveteliPro::InvoicesController < AlaveteliPro::BaseController + def index + @invoices = current_user.pro_account.invoices + end +end diff --git a/app/controllers/alaveteli_pro/payment_methods_controller.rb b/app/controllers/alaveteli_pro/payment_methods_controller.rb index 770bd3cf75..9d310042ea 100644 --- a/app/controllers/alaveteli_pro/payment_methods_controller.rb +++ b/app/controllers/alaveteli_pro/payment_methods_controller.rb @@ -6,12 +6,13 @@ def update @token = Stripe::Token.retrieve(params[:stripe_token]) @pro_account = current_user.pro_account ||= current_user.build_pro_account - @pro_account.source = @token.id + @pro_account.token = @token @pro_account.update_stripe_customer flash[:notice] = _('Your payment details have been updated') - rescue Stripe::CardError => e + rescue ProAccount::CardError, + Stripe::CardError => e flash[:error] = e.message rescue Stripe::RateLimitError, diff --git a/app/controllers/alaveteli_pro/stripe_webhooks_controller.rb b/app/controllers/alaveteli_pro/stripe_webhooks_controller.rb index 4bb9f98187..957b7d89fe 100644 --- a/app/controllers/alaveteli_pro/stripe_webhooks_controller.rb +++ b/app/controllers/alaveteli_pro/stripe_webhooks_controller.rb @@ -38,7 +38,7 @@ def receive store_unhandled_webhook end - # send a 200 ok to acknowlege receipt of the webhook + # send a 200 ok to acknowledge receipt of the webhook # https://stripe.com/docs/webhooks#responding-to-a-webhook render json: { message: 'OK' }, status: 200 end diff --git a/app/controllers/alaveteli_pro/subscriptions_controller.rb b/app/controllers/alaveteli_pro/subscriptions_controller.rb index 927f527318..cf40483e1e 100644 --- a/app/controllers/alaveteli_pro/subscriptions_controller.rb +++ b/app/controllers/alaveteli_pro/subscriptions_controller.rb @@ -46,7 +46,7 @@ def create @token = Stripe::Token.retrieve(params[:stripe_token]) - @pro_account.source = @token.id + @pro_account.token = @token @pro_account.update_stripe_customer @subscription = @pro_account.subscriptions.build @@ -60,7 +60,8 @@ def create @subscription.save - rescue Stripe::CardError => e + rescue ProAccount::CardError, + Stripe::CardError => e flash[:error] = e.message rescue Stripe::RateLimitError, diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb index 6dd05e2365..64c5ed2ba5 100644 --- a/app/controllers/api_controller.rb +++ b/app/controllers/api_controller.rb @@ -208,14 +208,14 @@ def body_request_events @events = InfoRequestEvent.where(event_type_clause). joins(:info_request). where("public_body_id = ?", @public_body.id). - includes([{info_request: :user}, :outgoing_message]). + includes([{ info_request: :user }, :outgoing_message]). order(created_at: :desc) if since_date_str begin since_date = Date.strptime(since_date_str, "%Y-%m-%d") rescue ArgumentError - render json: {"errors" => [ + render json: { "errors" => [ "Parameter since_date must be in format yyyy-mm-dd (not '#{since_date_str}')" ] }, status: 500 @@ -230,7 +230,7 @@ def body_request_events begin event = InfoRequestEvent.find(since_event_id) rescue ActiveRecord::RecordNotFound - render json: {"errors" => [ + render json: { "errors" => [ "Event ID #{since_event_id} not found" ] }, status: 500 diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 24f422e993..260d379e1f 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -404,7 +404,7 @@ def perform_search(models, query, sortby, collapse, per_page = 25, this_page = n # Work out sorting method order, ascending = order_to_sort_by(@sortby) - # Peform the search + # Perform the search @per_page = per_page @page = this_page || get_search_page_from_params diff --git a/app/controllers/attachment_masks_controller.rb b/app/controllers/attachment_masks_controller.rb new file mode 100644 index 0000000000..f5e349fa56 --- /dev/null +++ b/app/controllers/attachment_masks_controller.rb @@ -0,0 +1,57 @@ +## +# Controller to process FoiAttachment objects before being served publicly by +# applying masks and censor rules. +# +class AttachmentMasksController < ApplicationController + before_action :set_no_crawl_headers + before_action :decode_referer, :ensure_referer + before_action :find_attachment, :ensure_attachment + + def wait + if @attachment.masked? + redirect_to done_attachment_mask_path( + id: @attachment.to_signed_global_id, + referer: verifier.generate(@referer) + ) + + else + FoiAttachmentMaskJob.perform_once_later(@attachment) + end + end + + def done + unless @attachment.masked? + redirect_to wait_for_attachment_mask_path( + id: @attachment.to_signed_global_id, + referer: verifier.generate(@referer) + ) + end + end + + private + + def set_no_crawl_headers + headers['X-Robots-Tag'] = 'noindex' + end + + def decode_referer + @referer = verifier.verified(params[:referer]) + end + + def find_attachment + @attachment = GlobalID::Locator.locate_signed(params[:id]) + rescue ActiveRecord::RecordNotFound + end + + def ensure_referer + raise RouteNotFound unless @referer + end + + def ensure_attachment + redirect_to(@referer) unless @attachment + end + + def verifier + Rails.application.message_verifier('AttachmentsController') + end +end diff --git a/app/controllers/attachments_controller.rb b/app/controllers/attachments_controller.rb index 0dc2a2b747..3048407f43 100644 --- a/app/controllers/attachments_controller.rb +++ b/app/controllers/attachments_controller.rb @@ -16,24 +16,28 @@ class AttachmentsController < ApplicationController before_action :authenticate_attachment before_action :authenticate_attachment_as_html, only: :show_as_html - around_action :cache_attachments + around_action :cache_attachments, only: :show_as_html def show - # Prevent spam to magic request address. Note that the binary - # substitution method used depends on the content type - body = @incoming_message.apply_masks( - @attachment.default_body, - @attachment.content_type - ) + if @attachment.masked? + render body: @attachment.body, content_type: content_type + else + FoiAttachmentMaskJob.perform_once_later(@attachment) - if content_type == 'text/html' - body = - Loofah.scrub_document(body, :prune). - to_html(encoding: 'UTF-8'). - try(:html_safe) + Timeout.timeout(5) do + until @attachment.masked? + sleep 0.5 + @attachment.reload + end + redirect_to(request.fullpath) + end end - render body: body, content_type: content_type + rescue Timeout::Error + redirect_to wait_for_attachment_mask_path( + @attachment.to_signed_global_id, + referer: verifier.generate(request.fullpath) + ) end def show_as_html @@ -220,4 +224,8 @@ def prominence def with_prominence @info_request end + + def verifier + Rails.application.message_verifier('AttachmentsController') + end end diff --git a/app/controllers/concerns/user_spam_check.rb b/app/controllers/concerns/user_spam_check.rb index cbadf9c83f..4c599ba45e 100644 --- a/app/controllers/concerns/user_spam_check.rb +++ b/app/controllers/concerns/user_spam_check.rb @@ -48,7 +48,7 @@ def spam_scorer_config about_me_includes_anchor_tag?: 0, about_me_already_exists?: 0, user_agent_is_suspicious?: 3, - ip_range_is_suspicious?: 10 + ip_range_is_suspicious?: 12 } } end diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index f641ef151e..b5918ed050 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -161,7 +161,7 @@ def search @max_users = (@xapian_users.matches_estimated > MAX_RESULTS) ? MAX_RESULTS : @xapian_users.matches_estimated end - # Spelling and highight words are same for all three queries + # Spelling and highlight words are same for all three queries @highlight_words = @request_for_spelling.words_to_highlight(regex: true, include_original: true) unless @request_for_spelling.spelling_correction =~ /[a-z]+:/ @spelling_correction = @request_for_spelling.spelling_correction diff --git a/app/controllers/refusal_advice_controller.rb b/app/controllers/refusal_advice_controller.rb index 0d28b2333c..b882045b56 100644 --- a/app/controllers/refusal_advice_controller.rb +++ b/app/controllers/refusal_advice_controller.rb @@ -57,7 +57,7 @@ def help_page_redirect def external_redirect external = action.target[:external] - redirect_to external if external + redirect_to(external, allow_other_host: true) if external end def action diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index e4692be147..fdfb7866d7 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -371,6 +371,12 @@ def upload_response return false end + if @info_request.allow_new_responses_from == 'nobody' + render template: + 'request/request_subtitle/allow_new_responses_from/_nobody' + return + end + unless @info_request.public_body.is_foi_officer?(@user) domain_required = @info_request.public_body.foi_officer_domain_required if domain_required.nil? @@ -604,7 +610,7 @@ def make_request_summary_file(info_request) end def render_new_compose - params[:info_request] = { } unless params[:info_request] + params[:info_request] = {} unless params[:info_request] # Reconstruct the params # first the public body (by URL name or id) diff --git a/app/controllers/track_controller.rb b/app/controllers/track_controller.rb index 61dddb443a..b89a5331ed 100644 --- a/app/controllers/track_controller.rb +++ b/app/controllers/track_controller.rb @@ -1,5 +1,5 @@ # app/controllers/track_controller.rb: -# Publically visible email alerts and RSS - think an alert system crossed with +# Publicly visible email alerts and RSS - think an alert system crossed with # social bookmarking. # # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. diff --git a/app/controllers/user_controller.rb b/app/controllers/user_controller.rb index f64373a247..625b827b7d 100644 --- a/app/controllers/user_controller.rb +++ b/app/controllers/user_controller.rb @@ -542,7 +542,7 @@ def assign_request_states(display_user) end def set_display_user - User.find_by!(url_name: params[:url_name], email_confirmed: true) + User.where(email_confirmed: true).friendly.find(params[:url_name]) end def set_show_requests diff --git a/app/controllers/users/messages_controller.rb b/app/controllers/users/messages_controller.rb index 93a66ffdfd..90220a5c65 100644 --- a/app/controllers/users/messages_controller.rb +++ b/app/controllers/users/messages_controller.rb @@ -56,7 +56,7 @@ def check_can_send_messages def check_logged_in # You *must* be logged into send a message to another user. (This is - # partly to avoid spam, and partly to have some equanimity of openess + # partly to avoid spam, and partly to have some equanimity of openness # between the two users) # # "authenticated?" has done the redirect to signin page for us diff --git a/app/controllers/users/names_controller.rb b/app/controllers/users/names_controller.rb new file mode 100644 index 0000000000..59546f6bfe --- /dev/null +++ b/app/controllers/users/names_controller.rb @@ -0,0 +1,41 @@ +## +# Controller to allow current user to change their name +# +class Users::NamesController < ApplicationController + before_action :check_user_logged_in, :check_user_suspension, :load_user + + def update + if @edit_user.update(user_params) + flash[:notice] = _('Name successfully updated.') + redirect_to user_url(@edit_user) + else + render action: 'edit' + end + end + + private + + def user_params + params.require(:user).permit(:name) + end + + def check_user_logged_in + return if authenticated? + + flash[:error] = _('You need to be logged in to change your name') + redirect_to frontpage_url + end + + def check_user_suspension + return unless current_user.suspended? + + flash[:error] = _('Suspended users cannot edit their profile') + redirect_to edit_profile_about_me_path + end + + def load_user + # Don't make changes to the current_user, this could brake the layout as we + # use name/url_name in the login bar URLs + @edit_user = User.find_by(url_name: current_user.url_name) + end +end diff --git a/app/helpers/admin_general_timeline_helper.rb b/app/helpers/admin_general_timeline_helper.rb new file mode 100644 index 0000000000..c6fa3d70ef --- /dev/null +++ b/app/helpers/admin_general_timeline_helper.rb @@ -0,0 +1,36 @@ +## +# Helpers for rendering timeline filter buttons. Also used in the controller for +# generating the queries to load events. +# +module AdminGeneralTimelineHelper + def start_date + params[:start_date]&.to_datetime || 2.days.ago + end + + def time_filters + { + 'Hour' => 1.hour.ago, + 'Day' => 1.day.ago, + '2 days' => 2.days.ago, + 'Week' => 1.week.ago, + 'Month' => 1.month.ago, + 'All time' => Time.utc(1970, 1, 1) + } + end + + def current_time_filter + time_filters.min_by { |_, time| (time - start_date).abs }.first + end + + def event_types + { + authority_change: 'Authority changes', + info_request_event: 'Request events', + all: 'All events' + } + end + + def current_event_type + event_types[params[:event_type]&.to_sym] || event_types[:all] + end +end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 327427227a..60cf7abc5d 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -120,7 +120,7 @@ def theme_asset_exists?(asset_path) end # Note that if the admin interface is proxied via another server, we can't - # rely on a sesssion being shared between the front end and admin interface, + # rely on a session being shared between the front end and admin interface, # so need to check the status of the user. def is_admin? !session[:using_admin].nil? || (!@user.nil? && @user.is_admin?) diff --git a/app/helpers/link_to_helper.rb b/app/helpers/link_to_helper.rb index 522e4c33a1..ff0a6fd102 100755 --- a/app/helpers/link_to_helper.rb +++ b/app/helpers/link_to_helper.rb @@ -11,7 +11,7 @@ module LinkToHelper # Requests def request_url(info_request, options = {}) - show_request_url({url_title: info_request.url_title}.merge(options)) + show_request_url({ url_title: info_request.url_title }.merge(options)) end def request_path(info_request, options = {}) @@ -121,12 +121,14 @@ def user_path(user, options = {}) user_url(user, options.merge(only_path: true)) end - def user_link_absolute(user) - link_to user.name, user_url(user) + def user_link_absolute(user, text = nil) + text ||= user.name + link_to text, user_url(user) end - def user_link(user) - link_to user.name, user_path(user) + def user_link(user, text = nil) + text ||= user.name + link_to text, user_path(user) end def user_link_for_request(request) @@ -172,7 +174,7 @@ def request_user_link_absolute(request, anonymous_text = nil) if request.is_external? external_user_link_absolute(request, anonymous_text) else - user_link_absolute(request.user) + user_link_absolute(request.user, request.safe_from_name) end end @@ -180,7 +182,7 @@ def request_user_link(request, anonymous_text = nil) if request.is_external? external_user_link(request, anonymous_text) else - user_link(request.user) + user_link(request.user, request.safe_from_name) end end @@ -237,10 +239,10 @@ def search_url(query, options = nil) query -= ["", nil] query = query.join("/") end - routing_info = {controller: 'general', + routing_info = { controller: 'general', action: 'search', combined: query, - view: nil} + view: nil } routing_info = options.merge(routing_info) unless options.nil? if routing_info.is_a?(Hash) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 7a81d6b20f..06eafd1c73 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -5,9 +5,4 @@ class ApplicationJob < ActiveJob::Base # :nodoc: # Most jobs are safe to ignore if the underlying records are no longer # available discard_on ActiveJob::DeserializationError - - def self.perform_later(*args) - return super unless AlaveteliConfiguration.background_jobs == 'inline' - perform_now(*args) - end end diff --git a/app/jobs/foi_attachment_mask_job.rb b/app/jobs/foi_attachment_mask_job.rb new file mode 100644 index 0000000000..e5756653f0 --- /dev/null +++ b/app/jobs/foi_attachment_mask_job.rb @@ -0,0 +1,66 @@ +## +# Job to apply masks and censor rules to FoiAttachment objects. Masked file will +# be stored as FoiAttachment#file ActiveStorage association. +# +# Example: +# FoiAttachmentMaskJob.perform(FoiAttachment.first) +# +class FoiAttachmentMaskJob < ApplicationJob + include Uniqueness + + queue_as :default + + attr_reader :attachment + + delegate :incoming_message, to: :attachment + delegate :info_request, to: :incoming_message + + def perform(attachment) + @attachment = attachment + mask + + rescue FoiAttachment::MissingAttachment + incoming_message.parse_raw_email!(true) + + begin + attachment.reload + rescue ActiveRecord::RecordNotFound + @attachment = attachment.load_attachment_from_incoming_message + end + + mask + end + + private + + def mask + body = AlaveteliTextMasker.apply_masks( + attachment.unmasked_body, + attachment.content_type, + masks + ) + + if attachment.content_type == 'text/html' + body = + Loofah.scrub_document(body, :prune). + to_html(encoding: 'UTF-8'). + try(:html_safe) + end + + attachment.update(body: body, masked_at: Time.zone.now) + + # ensure the after_commit callback runs which uploads the blob, without this + # the callback might not execute in time and the job exits resulting in the + # lost of the masked attachment body. + return if attachment.file_blob.service.exist?(attachment.file_blob.key) + + attachment.run_callbacks(:commit) + end + + def masks + { + censor_rules: info_request.applicable_censor_rules, + masks: info_request.masks + } + end +end diff --git a/app/jobs/foi_attachment_mask_job/uniqueness.rb b/app/jobs/foi_attachment_mask_job/uniqueness.rb new file mode 100644 index 0000000000..6579727ff9 --- /dev/null +++ b/app/jobs/foi_attachment_mask_job/uniqueness.rb @@ -0,0 +1,34 @@ +## +# Module to add methods to check for existing jobs before performing/enqueuing +# attempting to ensure we execute once only. +# +# These methods only work if Sidekiq is used as the ActiveJob adapter. +# +module FoiAttachmentMaskJob::Uniqueness + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + def perform_once_later(attachment) + perform_later(attachment) unless existing_job(attachment) + end + + def perform_once_now(attachment) + existing_job(attachment)&.delete + perform_now(attachment) + end + + def existing_job(attachment) + return unless queue_adapter.is_a?( + ActiveJob::QueueAdapters::SidekiqAdapter + ) + + queue = Sidekiq::Queue.new(queue_name) + queue.find do |j| + gid = j.display_args.first['_aj_globalid'] + gid == attachment.to_gid.to_s + end + end + end +end diff --git a/app/jobs/info_request_expire_job.rb b/app/jobs/info_request_expire_job.rb index 394d21bf1f..a514f520df 100644 --- a/app/jobs/info_request_expire_job.rb +++ b/app/jobs/info_request_expire_job.rb @@ -8,7 +8,7 @@ # InfoRequestExpireJob.perform(PublicBody.first, :info_requests) # class InfoRequestExpireJob < ApplicationJob - queue_as :default + queue_as :xapian def perform(object, method = nil) return object.expire if object.is_a?(InfoRequest) diff --git a/app/jobs/notify_cache_job.rb b/app/jobs/notify_cache_job.rb new file mode 100644 index 0000000000..f5d77f173a --- /dev/null +++ b/app/jobs/notify_cache_job.rb @@ -0,0 +1,68 @@ +require 'net/http' + +class Net::HTTP::Purge < Net::HTTP::Get + METHOD = 'PURGE' +end + +class Net::HTTP::Ban < Net::HTTP::Get + METHOD = 'BAN' +end + +## +# Job to notify a cache of URLs to be purged or banned, given an object +# (that must have a cached_urls method). +# +# Examples: +# NotifyCacheJob.perform(InfoRequest.first) +# NotifyCacheJob.perform(FoiAttachment.first) +# NotifyCacheJob.perform(Comment.first) +# +class NotifyCacheJob < ApplicationJob + queue_as :default + + around_enqueue do |_, block| + block.call if AlaveteliConfiguration.varnish_hosts.present? + end + + def perform(object) + urls = object.cached_urls + locales = [''] + AlaveteliLocalization.available_locales.map { "/#{_1}" } + hosts = AlaveteliConfiguration.varnish_hosts + + urls.product(locales, hosts).each do |url, locale, host| + if url.start_with? '^' + request = Net::HTTP::Ban.new('/') + request['X-Invalidate-Pattern'] = '^' + locale + url[1..-1] + else + request = Net::HTTP::Purge.new(locale + url) + end + + response = connection_for_host(host).request(request) + log_result = "#{request.method} #{url} at #{host}: #{response.code}" + + case response + when Net::HTTPSuccess + Rails.logger.debug('NotifyCacheJob: ' + log_result) + else + Rails.logger.warn('NotifyCacheJob: Unable to ' + log_result) + end + end + + ensure + close_connections + end + + def connections + @connections ||= {} + end + + def connection_for_host(host) + connections[host] ||= Net::HTTP.start( + AlaveteliConfiguration.domain, 80, host, 6081 + ) + end + + def close_connections + connections.values.each { _1.finish if _1.started? } + end +end diff --git a/app/mailers/request_mailer.rb b/app/mailers/request_mailer.rb index a256c71e52..499f8ae43d 100644 --- a/app/mailers/request_mailer.rb +++ b/app/mailers/request_mailer.rb @@ -24,8 +24,8 @@ def fake_response(info_request, from_user, message_body, attachment_name, attach if !attachment_name.nil? && !attachment_content.nil? content_type = AlaveteliFileTypes.filename_to_mimetype(attachment_name) || 'application/octet-stream' - attachments[attachment_name] = {content: attachment_content, - content_type: content_type} + attachments[attachment_name] = { content: attachment_content, + content_type: content_type } end mail(from: from_user.name_and_email, @@ -38,8 +38,8 @@ def external_response(info_request, message_body, sent_at, attachment_hashes) @message_body = message_body attachment_hashes.each do |attachment_hash| - attachments[attachment_hash[:filename]] = {content: attachment_hash[:body], - content_type: attachment_hash[:content_type]} + attachments[attachment_hash[:filename]] = { content: attachment_hash[:body], + content_type: attachment_hash[:content_type] } end mail(from: blackhole_email, @@ -231,40 +231,53 @@ def self.receive(raw_email, source = :mailin) # Find which info requests the email is for def requests_matching_email(email) - # We deliberately don't use Envelope-to here, so ones that are BCC - # drop into the holding pen for checking. - addresses = ((email.to || []) + (email.cc || [])).compact + addresses = MailHandler.get_all_addresses(email) InfoRequest.matching_incoming_email(addresses) end + def send_to_holding_pen(email, raw_email, opts) + opts[:rejected_reason] = + _("Could not identify the request from the email address") + request = InfoRequest.holding_pen_request + request.receive(email, raw_email, opts) + end + # Member function, called on the new class made in self.receive above def receive(email, raw_email, source = :mailin) opts = { source: source } - # Find which info requests the email is for - reply_info_requests = requests_matching_email(email) + # Only check mail that doesn't have spam in the header + return if SpamAddress.spam?(MailHandler.get_all_addresses(email)) - # Nothing found, so save in holding pen - if reply_info_requests.empty? - opts[:rejected_reason] = - _("Could not identify the request from the email address") - request = InfoRequest.holding_pen_request + # Find exact matches for info requests + exact_info_requests = requests_matching_email(email) - request.receive(email, raw_email, opts) unless SpamAddress.spam?(email.to) + if exact_info_requests.count > 0 + # Go through each exact info request and deliver the email + exact_info_requests.each do |info_request| + info_request.receive(email, raw_email, opts) + end return end - # Send the message to each request, to be archived with it - reply_info_requests.each do |reply_info_request| - # If environment variable STOP_DUPLICATES is set, don't send message with same id again - if ENV['STOP_DUPLICATES'] - if reply_info_request.already_received?(email, raw_email) - raise "message #{ email.message_id } already received by request" - end - end - - reply_info_request.receive(email, raw_email, opts) + # If there are no exact matches, find any guessed requests + guessed_info_requests = Guess.guessed_info_requests(email) + + if guessed_info_requests.count == 1 + # If there one guess automatically redeliver the email to that and log it + # as an event + info_request = guessed_info_requests.first + info_request.log_event( + 'redeliver_incoming', + editor: 'automatic', + destination_request: info_request + ) + info_request.receive(email, raw_email, opts) + + else + # Otherwise we send the mail to the holding pen + send_to_holding_pen(email, raw_email, opts) end end diff --git a/app/models/alaveteli_pro/embargo.rb b/app/models/alaveteli_pro/embargo.rb index 6e06e65715..864414b343 100644 --- a/app/models/alaveteli_pro/embargo.rb +++ b/app/models/alaveteli_pro/embargo.rb @@ -124,7 +124,7 @@ def self.log_expiring_events AND ire.created_at = embargoes.expiring_notification_at AND ire.event_type = 'embargo_expiring'" embargoes = expiring.joins(query).where("ire.info_request_id IS NULL") - embargoes.find_each do |embargo| + embargoes.find_each(batch_size: 200) do |embargo| info_request = embargo.info_request event = info_request.log_event( 'embargo_expiring', diff --git a/app/models/alaveteli_pro/invoice.rb b/app/models/alaveteli_pro/invoice.rb new file mode 100644 index 0000000000..b982ba10e8 --- /dev/null +++ b/app/models/alaveteli_pro/invoice.rb @@ -0,0 +1,37 @@ +module AlaveteliPro + ## + # This class adds wraps a Stripe::Invoice object to customise behaviour + # and to add useful helper methods. + # + class Invoice < SimpleDelegator + # state + def open? + status == 'open' + end + + def paid? + status == 'paid' + end + + # attributes + def date + Time.at(super).to_date + end + + # charge + def charge + @charge ||= Stripe::Charge.retrieve(__getobj__.charge) + end + + delegate :receipt_url, to: :charge + + private + + def method_missing(*args) + # Forward missing methods such as #coupon= as on a blank subscription + # this wouldn't be delegated due to how Stripe::APIResource instances + # use meta programming to dynamically define setting methods. + __getobj__.public_send(*args) + end + end +end diff --git a/app/models/alaveteli_pro/invoice_collection.rb b/app/models/alaveteli_pro/invoice_collection.rb new file mode 100644 index 0000000000..74a8dc4732 --- /dev/null +++ b/app/models/alaveteli_pro/invoice_collection.rb @@ -0,0 +1,56 @@ +module AlaveteliPro + ## + # This class is responsible for loading and wrapping Stripe invoices as + # AlaveteliPro::Invoice objects. This allows us to easily customise + # behaviour and add helper methods. + # + class InvoiceCollection + include Enumerable + + def self.for_customer(customer) + new(customer) + end + + def initialize(customer) + @customer = customer + end + + def retrieve(id) + return unless @customer + AlaveteliPro::Invoice.new(invoices.retrieve(id)) + end + + # scope + def open + select(&:open?) + end + + def paid + select(&:paid?) + end + + # enumerable + def each(&block) + if block_given? + wrapped_block = -> (invoice) do + block.call(AlaveteliPro::Invoice.new(invoice)) + end + + if invoices.is_a?(Stripe::ListObject) + invoices.auto_paging_each(&wrapped_block) + else + invoices.each(&wrapped_block) + end + else + to_enum(:each) + end + end + + private + + def invoices + return [] unless @customer + @invoices ||= Stripe::Invoice.list(customer: @customer) + end + end +end diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index a3704e9b6c..5a40589cc4 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -75,6 +75,7 @@ def is_global? def expire_requests if info_request InfoRequestExpireJob.perform_later(info_request) + NotifyCacheJob.perform_later(info_request) elsif user InfoRequestExpireJob.perform_later(user, :info_requests) elsif public_body diff --git a/app/models/comment.rb b/app/models/comment.rb index 07b295d271..747a66a7df 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -193,6 +193,13 @@ def hide(editor:) end end + def cached_urls + [ + request_path(info_request), + show_user_wall_path(url_name: user.url_name) + ] + end + private def check_body_has_content diff --git a/app/models/dataset/value_set.rb b/app/models/dataset/value_set.rb index c399ca4d8b..8e452c9b88 100644 --- a/app/models/dataset/value_set.rb +++ b/app/models/dataset/value_set.rb @@ -36,6 +36,6 @@ class Dataset::ValueSet < ApplicationRecord def check_at_least_one_value_is_present return unless values.map(&:value).all?(&:blank?) - errors.add :values, :emtpy + errors.add :values, :empty end end diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index f67986eed4..bf196f53e1 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 20220916134847 +# Schema version: 20230717201410 # # Table name: foi_attachments # @@ -16,6 +16,7 @@ # updated_at :datetime # prominence :string default("normal") # prominence_reason :text +# masked_at :datetime # # models/foi_attachment.rb: @@ -28,10 +29,15 @@ require 'digest' class FoiAttachment < ApplicationRecord + include Rails.application.routes.url_helpers + include LinkToHelper include MessageProminence + MissingAttachment = Class.new(StandardError) + belongs_to :incoming_message, inverse_of: :foi_attachments + has_one :raw_email, through: :incoming_message, source: :raw_email has_one_attached :file, service: :attachments @@ -49,13 +55,39 @@ class FoiAttachment < ApplicationRecord BODY_MAX_TRIES = 3 BODY_MAX_DELAY = 5 + # rubocop:disable Style/LineLength + CONTENT_TYPE_NAMES = { + # Plain Text + "text/plain" => 'Text file', + 'application/rtf' => 'RTF file', + + # Binary Documents + 'application/pdf' => 'PDF file', + + # Images + 'image/tiff' => 'TIFF image', + + # Word Processing + 'application/vnd.ms-word' => 'Word document', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'Word document', + + # Presentation + 'application/vnd.ms-powerpoint' => 'PowerPoint presentation', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'PowerPoint presentation', + + # Spreadsheet + 'application/vnd.ms-excel' => 'Excel spreadsheet', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'Excel spreadsheet' + }.freeze + # rubocop:enable Style/LineLength + def delete_cached_file! @cached_body = nil file.purge if file.attached? end def body=(d) - self.hexdigest = Digest::MD5.hexdigest(d) + self.hexdigest ||= Digest::MD5.hexdigest(d) ensure_filename! file.attach( @@ -69,21 +101,19 @@ def body=(d) end # raw body, encoded as binary - def body(tries: 0, delay: 1) + def body return @cached_body if @cached_body - if file.attached? + if masked? @cached_body = file.download - else - # we've lost our cached attachments for some reason. Reparse them. - raise if tries > BODY_MAX_TRIES - sleep [delay, BODY_MAX_DELAY].min - - incoming_message.parse_raw_email!(true) + elsif persisted? + FoiAttachmentMaskJob.perform_once_now(self) reload - - body(tries: tries + 1, delay: delay * 2) + body end + + rescue ActiveRecord::RecordNotFound + load_attachment_from_incoming_message.body end # body as UTF-8 text, with scrubbing of invalid chars if needed @@ -97,6 +127,31 @@ def default_body text_type? ? body_as_text.string : body end + # return the body as it is in the raw email, unmasked without censor rules + # applied + def unmasked_body + MailHandler.attachment_body_for_hexdigest( + raw_email.mail, + hexdigest: hexdigest + ) + rescue MailHandler::MismatchedAttachmentHexdigest + attributes = MailHandler.attempt_to_find_original_attachment_attributes( + raw_email.mail, + body: file.download + ) if file.attached? + + unless attributes + raise MissingAttachment, "attachment missing in raw email (ID=#{id})" + end + + update(hexdigest: attributes[:hexdigest]) + attributes[:body] + end + + def masked? + file.attached? && masked_at.present? && masked_at < Time.zone.now + end + def main_body_part? self == incoming_message.get_main_body_text_part end @@ -241,51 +296,15 @@ def update_display_size! end end - # Whether this type can be shown in the Google Docs Viewer. - # The full list of supported types can be found at - # https://docs.google.com/support/bin/answer.py?hl=en&answer=1189935 - def has_google_docs_viewer? - [ - "application/pdf", # .pdf - "image/tiff", # .tiff - - "application/vnd.ms-word", # .doc - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", # .docx - - "application/vnd.ms-powerpoint", # .ppt - "application/vnd.openxmlformats-officedocument.presentationml.presentation", # .pptx - - "application/vnd.ms-excel", # .xls - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" # .xlsx - ].include?(content_type) - end - # Whether this type has a "View as HTML" def has_body_as_html? - [ - "text/plain", - "application/rtf" - ].include?(content_type) || has_google_docs_viewer? + AttachmentToHTML.extractable?(self) end - # Name of type of attachment type - only valid for things that has_body_as_html? + # Name of type of attachment type - only valid for things that + # has_body_as_html? def name_of_content_type - { - "text/plain" => "Text file", - 'application/rtf' => "RTF file", - - 'application/pdf' => "PDF file", - 'image/tiff' => "TIFF image", - - 'application/vnd.ms-word' => "Word document", - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => "Word document", - - 'application/vnd.ms-powerpoint' => "PowerPoint presentation", - 'application/vnd.openxmlformats-officedocument.presentationml.presentation' => "PowerPoint presentation", - - 'application/vnd.ms-excel' => "Excel spreadsheet", - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => "Excel spreadsheet" - }[content_type] + CONTENT_TYPE_NAMES[content_type] end # For "View as HTML" of attachment @@ -295,10 +314,23 @@ def body_as_html(dir, opts = {}) AttachmentToHTML.to_html(self, to_html_opts) end + def cached_urls + [ + request_path(incoming_message.info_request) + ] + end + + def load_attachment_from_incoming_message + IncomingMessage.get_attachment_by_url_part_number_and_filename!( + incoming_message.get_attachments_for_display, + url_part_number, + display_filename + ) + end + private def text_type? AlaveteliTextMasker::TextMask.include?(content_type) end - end diff --git a/app/models/guess.rb b/app/models/guess.rb new file mode 100644 index 0000000000..f75eafbf70 --- /dev/null +++ b/app/models/guess.rb @@ -0,0 +1,64 @@ +require 'text' + +## +# A guess at which info request a incoming message should be associated to +# +class Guess + attr_reader :info_request, :components + + # The percentage similarity the id or idhash much fulfil + THRESHOLD = 0.8 + + ## + # Return InfoRequest which we guess should receive an incoming message based + # on a threshold. + # + def self.guessed_info_requests(email) + # Match the email address in the message without matching the hash + email_addresses = MailHandler.get_all_addresses(email) + guesses = InfoRequest.guess_by_incoming_email(email_addresses) + + guesses_reaching_threshold = guesses.select do |ir_guess| + id_score = ir_guess.id_score + idhash_score = ir_guess.idhash_score + + (id_score == 1 && idhash_score >= THRESHOLD) || + (id_score >= THRESHOLD && idhash_score == 1) + end + + guesses_reaching_threshold.map(&:info_request).uniq + end + + def initialize(info_request, **components) + @info_request = info_request + @components = components + end + + def [](key) + components[key] + end + + def id_score + return 1 unless self[:id] + similarity(self[:id], info_request.id) + end + + def idhash_score + return 1 unless self[:idhash] + similarity(self[:idhash], info_request.idhash) + end + + def ==(other) + info_request == other.info_request && components == other.components + end + + def match_method + components.keys.first + end + + private + + def similarity(a, b) + Text::WhiteSimilarity.similarity(a.to_s, b.to_s) + end +end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index d15d8bb950..2b2ba2b1b8 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -41,6 +41,8 @@ class IncomingMessage < ApplicationRecord include CacheAttributesFromRawEmail include Taggable + UnableToExtractAttachments = Class.new(StandardError) + MAX_ATTACHMENT_TEXT_CLIPPED = 1_000_000 # 1Mb ish belongs_to :info_request, @@ -416,28 +418,10 @@ def get_main_body_text_part(leaves=[]) # Returns attachments that are uuencoded in main body part def _uudecode_attachments(text, start_part_number) - # Find any uudecoded things buried in it, yeuchly - uus = text.scan(/^begin.+^`\n^end\n/m) - uus.map.with_index do |uu, index| - # Decode the string - content = uu.sub(/\Abegin \d+ [^\n]*\n/, '').unpack('u').first - # Make attachment type from it, working out filename and mime type - filename = uu.match(/^begin\s+[0-9]+\s+(.*)$/)[1] - calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(filename, content) - if calc_mime - calc_mime = MailHandler.normalise_content_type(calc_mime) - content_type = calc_mime - else - content_type = 'application/octet-stream' - end - hexdigest = Digest::MD5.hexdigest(content) + MailHandler.uudecode(text, start_part_number).map do |attrs| + hexdigest = attrs.delete(:hexdigest) attachment = foi_attachments.find_or_initialize_by(hexdigest: hexdigest) - attachment.attributes = { - filename: filename, - content_type: content_type, - body: content, - url_part_number: start_part_number + index + 1 - } + attachment.attributes = attrs attachment end end @@ -462,6 +446,7 @@ def extract_attachments _mail = raw_email.mail! attachment_attributes = MailHandler.get_attachment_attributes(_mail) attachment_attributes = attachment_attributes.inject({}) do |memo, attrs| + attrs.delete(:original_body) memo[attrs[:hexdigest]] = attrs memo end @@ -487,8 +472,19 @@ def extract_attachments attachments += _uudecode_attachments(main_part.body, c) end - # Purge old attachments that have been rebuilt with a new hexdigest - (foi_attachments - attachments).each(&:mark_for_destruction) + # Purge old public attachments that will be rebuilt with a new hexdigest + old_attachments = (foi_attachments - attachments) + hidden_old_attachments = old_attachments.reject { _1.is_public? } + + if hidden_old_attachments.any? + # if there are hidden attachments error as we don't want to re-build and + # lose the prominence as this will make them public + raise UnableToExtractAttachments, "unable to extract attachments due " \ + "to prominence of attachments " \ + "(ID=#{hidden_old_attachments.map(&:id).join(', ')})" + else + old_attachments.each(&:mark_for_destruction) + end end # Returns body text as HTML with quotes flattened, and emails removed. diff --git a/app/models/info_request.rb b/app/models/info_request.rb index b64c356986..f8892a742b 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -40,7 +40,6 @@ require 'fileutils' class InfoRequest < ApplicationRecord - Guess = Struct.new(:info_request, :matched_value, :match_method).freeze OLD_AGE_IN_DAYS = 21.days include Rails.application.routes.url_helpers @@ -52,6 +51,7 @@ class InfoRequest < ApplicationRecord include InfoRequest::TitleValidation include Taggable include Notable + include LinkToHelper admin_columns exclude: %i[title url_title], include: %i[rejected_incoming_count] @@ -244,8 +244,13 @@ def self.guess_by_incoming_email(*emails) guesses = emails.flatten.reduce([]) do |memo, email| id, idhash = _extract_id_hash_from_email(email) id, idhash = _guess_idhash_from_email(email) if idhash.nil? || id.nil? - memo << Guess.new(find_by_id(id), email, :id) - memo << Guess.new(find_by_idhash(idhash), email, :idhash) + + memo << Guess.new( + find_by_id(id), email: email, id: id, idhash: idhash + ) + memo << Guess.new( + find_by_idhash(idhash), email: email, id: id, idhash: idhash + ) end # Unique Guesses where we've found an `InfoRequest` @@ -314,7 +319,7 @@ def self.guess_by_incoming_subject(subject_line) limit(25) guesses = requests.each.reduce([]) do |memo, request| - memo << Guess.new(request, subject_line, :subject) + memo << Guess.new(request, subject: subject_line) end # Unique Guesses where we've found an `InfoRequest` @@ -369,7 +374,7 @@ def self.holding_pen_request om = OutgoingMessage.new({ status: 'ready', message_type: 'initial_request', - body: 'This is the holding pen request. It shows responses that were sent to invalid addresses, and need moving to the correct request by an adminstrator.', + body: 'This is the holding pen request. It shows responses that were sent to invalid addresses, and need moving to the correct request by an administrator.', last_sent_at: Time.zone.now, what_doing: 'normal_sort' @@ -542,12 +547,13 @@ def self.requests_very_old_after_months def self.stop_new_responses_on_old_requests # 'old' months since last change to request, only allow new incoming # messages from authority domains - InfoRequest - .been_published - .where(allow_new_responses_from: 'anybody') - .where.not(url_title: 'holding_pen') - .updated_before(requests_old_after_months.months.ago.to_date) - .find_in_batches do |batch| + InfoRequest. + been_published. + where(allow_new_responses_from: 'anybody'). + where.not(url_title: 'holding_pen'). + updated_before(requests_old_after_months.months.ago.to_date). + distinct. + find_in_batches do |batch| batch.each do |info_request| old_allow_new_responses_from = info_request.allow_new_responses_from @@ -565,12 +571,13 @@ def self.stop_new_responses_on_old_requests # 'very_old' months since last change to request, don't allow any new # incoming messages - InfoRequest - .been_published - .where(allow_new_responses_from: %w[anybody authority_only]) - .where.not(url_title: 'holding_pen') - .updated_before(requests_very_old_after_months.months.ago.to_date) - .find_in_batches do |batch| + InfoRequest. + been_published. + where(allow_new_responses_from: %w[anybody authority_only]). + where.not(url_title: 'holding_pen'). + updated_before(requests_very_old_after_months.months.ago.to_date). + distinct. + find_in_batches do |batch| batch.each do |info_request| old_allow_new_responses_from = info_request.allow_new_responses_from @@ -755,7 +762,18 @@ def is_external? end def user_name - is_external? ? external_user_name : user.name + return external_user_name if is_external? + user&.name + end + + def from_name + return external_user_name if is_external? + outgoing_messages.first&.from_name || user_name + end + + def safe_from_name + return external_user_name if is_external? + apply_censor_rules_to_text(from_name) end def user_name_slug @@ -796,6 +814,10 @@ def tag_string=(tag_string) end def expire(options={}) + # Clear any attachment masked_at timestamp, forcing attachments to be + # reparsed + clear_attachment_masks! + # Clear out cached entries, by removing files from disk (the built in # Rails fragment cache made doing this and other things too hard) foi_fragment_cache_directories.each { |dir| FileUtils.rm_rf(dir) } @@ -811,6 +833,10 @@ def expire(options={}) reindex_request_events end + def clear_attachment_masks! + foi_attachments.update_all(masked_at: nil) + end + # Removes anything cached about the object in the database, and saves def clear_in_database_caches! incoming_messages.each(&:clear_in_database_caches!) @@ -875,18 +901,14 @@ def find_existing_outgoing_message(body) end # Has this email already been received here? Based just on message id. - def already_received?(email, _raw_email_data) - message_id = email.message_id - raise "No message id for this message" if message_id.nil? - - incoming_messages.each do |im| - return true if message_id == im.message_id - end - - false + def already_received?(email) + return false unless email.message_id + incoming_messages.any? { email.message_id == _1.message_id } end def receive(email, raw_email_data, *args) + return if already_received?(email) + defaults = { override_stop_new_responses: false, rejected_reason: nil, source: :internal } @@ -1739,6 +1761,34 @@ def latest_refusals incoming_messages.select(&:refusals?).last&.refusals || [] end + def cached_urls + feed_request = TrackThing.new( + info_request: self, + track_type: 'request_updates' + ) + feed_body = TrackThing.new( + public_body: public_body, + track_type: 'public_body_updates' + ) + feed_user = TrackThing.new( + tracked_user: user, + track_type: 'user_updates' + ) + [ + '/', + public_body_path(public_body), + request_path(self), + request_details_path(self), + '^/list', + do_track_path(feed_request, feed = 'feed'), + '^/feed/list/', + do_track_path(feed_body, feed = 'feed'), + do_track_path(feed_user, feed = 'feed'), + user_path(user), + show_user_wall_path(url_name: user.url_name) + ] + end + private def self.add_conditions_from_extra_params(params, extra_params) diff --git a/app/models/info_request/pro_query.rb b/app/models/info_request/pro_query.rb index 4cab5280b8..46cde2defe 100644 --- a/app/models/info_request/pro_query.rb +++ b/app/models/info_request/pro_query.rb @@ -7,7 +7,7 @@ def initialize(relation = InfoRequest) def call @relation .includes(user: :roles) - .where(roles: {name: 'pro'}) + .where(roles: { name: 'pro' }) .references(:roles) end end diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index 3892b1a0a9..64a2cde8be 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -88,6 +88,7 @@ class InfoRequestEvent < ApplicationRecord self.event_type = "hide" end after_create :update_request, if: :response? + after_create :invalidate_cached_pages after_commit -> { info_request.create_or_update_request_summary }, on: [:create] @@ -261,7 +262,7 @@ def params_diff end end new_params.delete_if { |key, _value| ignore.keys.include?(key) } - {new: new_params, old: old_params, other: other_params} + { new: new_params, old: old_params, other: other_params } end def is_incoming_message? @@ -355,6 +356,16 @@ def update_request info_request.update_last_public_response_at end + def invalidate_cached_pages + if comment + NotifyCacheJob.perform_later(comment) + elsif foi_attachment + NotifyCacheJob.perform_later(foi_attachment) + else + NotifyCacheJob.perform_later(info_request) + end + end + def same_email_as_previous_send? prev_addr = info_request.get_previous_email_sent_to(self) curr_addr = params[:email] @@ -406,6 +417,11 @@ def set_calculated_state!(state) end end + def foi_attachment + return unless params[:attachment_id] + @foi_attachment ||= FoiAttachment.find(params[:attachment_id]) + end + protected def variety diff --git a/app/models/mail_server_log.rb b/app/models/mail_server_log.rb index 2144d2aacf..701df4e9f6 100644 --- a/app/models/mail_server_log.rb +++ b/app/models/mail_server_log.rb @@ -109,7 +109,7 @@ def self.scan_for_postfix_queue_ids(f) result end - # Retuns nil if there is no queue id + # Returns nil if there is no queue id def self.extract_postfix_queue_id_from_syslog_line(line) # Assume the log file was written using syslog and parse accordingly m = SyslogProtocol.parse("<13>" + line).content.match(/^\S+: (\S+):/) @@ -170,7 +170,7 @@ def self.request_postfix_sent?(ir) # no request- email in it? # # NB: There can be several emails involved in a request. This just checks that - # at least one of them has been succesfully sent. + # at least one of them has been successfully sent. # def self.check_recent_requests_have_been_sent # Get all requests sent for from 2 to 10 days ago. The 2 day gap is diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index 434384deb4..80a489226f 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 20210114161442 +# Schema version: 20230412084830 # # Table name: outgoing_messages # @@ -15,6 +15,7 @@ # what_doing :string not null # prominence :string default("normal"), not null # prominence_reason :text +# from_name :text # # models/outgoing_message.rb: @@ -40,7 +41,9 @@ class OutgoingMessage < ApplicationRecord # To override the default letter attr_accessor :default_letter + before_validation :cache_from_name validates_presence_of :info_request + validates_presence_of :from_name, unless: -> (m) { !m.info_request&.user } validates_inclusion_of :status, in: STATUS_TYPES validates_inclusion_of :message_type, in: MESSAGE_TYPES validate :template_changed @@ -55,6 +58,10 @@ class OutgoingMessage < ApplicationRecord foreign_key: 'incoming_message_followup_id', class_name: 'IncomingMessage' + has_one :user, + inverse_of: :outgoing_messages, + through: :info_request + # can have many events, for items which were resent by site admin e.g. if # contact address changed has_many :info_request_events, @@ -130,6 +137,16 @@ def set_signature_name(name) self.body = get_default_message + name if raw_body == get_default_message end + def from_name + return info_request.external_user_name if info_request.is_external? + super || info_request.user_name + end + + def safe_from_name + return info_request.external_user_name if info_request.is_external? + info_request.apply_censor_rules_to_text(from_name) + end + # Public: The value to be used in the From: header of an OutgoingMailer # message. # @@ -393,6 +410,11 @@ def default_letter=(text) private + def cache_from_name + return if read_attribute(:from_name) + self.from_name = info_request.user_name if info_request + end + def set_info_request_described_state if status == 'failed' info_request.set_described_state('error_message') diff --git a/app/models/pro_account.rb b/app/models/pro_account.rb index 32633e6440..d668a17847 100644 --- a/app/models/pro_account.rb +++ b/app/models/pro_account.rb @@ -14,7 +14,9 @@ class ProAccount < ApplicationRecord include AlaveteliFeatures::Helpers - attr_writer :source + CardError = Class.new(StandardError) + + attr_writer :token belongs_to :user, inverse_of: :pro_account @@ -33,6 +35,12 @@ def subscriptions ) end + def invoices + @invoices ||= AlaveteliPro::InvoiceCollection.for_customer( + stripe_customer + ) + end + def stripe_customer @stripe_customer ||= stripe_customer! end @@ -59,9 +67,9 @@ def update_email end def update_source - return unless @source + return unless @token - stripe_customer.source = @source + stripe_customer.source = @token.id end def stripe_customer! diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 8ec9dd5bcf..41b74a469e 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -33,8 +33,11 @@ require 'confidence_intervals' class PublicBody < ApplicationRecord + include CalculatedHomePage include Taggable include Notable + include Rails.application.routes.url_helpers + include LinkToHelper class ImportCSVDryRun < StandardError; end @@ -114,7 +117,7 @@ def self.admin_title after_save :update_missing_email_tag - after_update :reindex_requested_from + after_update :reindex_requested_from, :invalidate_cached_pages # Every public body except for the internal admin one is visible scope :visible, -> { where("public_bodies.id <> #{ PublicBody.internal_admin_body.id }") } @@ -136,7 +139,7 @@ def self.admin_title strip_attributes allow_empty: true, only: %i[request_email] translates :name, :short_name, :request_email, :url_name, :first_letter, - :publication_scheme + :publication_scheme, :disclosure_log # Cannot be grouped at top as it depends on the `translates` macro include Translatable @@ -230,7 +233,7 @@ def editor # tags contain the given query # # query - String to query the searchable fields - # locale - String to specify the language of the seach query + # locale - String to specify the language of the search query # (default: AlaveteliLocalization.locale) # # Returns an ActiveRecord::Relation @@ -400,19 +403,6 @@ def legislation legislations.first end - # Guess home page from the request email, or use explicit override, or nil - # if not known. - # - # TODO: PublicBody#calculated_home_page would be a good candidate to cache - # in an instance variable - def calculated_home_page - if home_page && !home_page.empty? - home_page[URI.regexp(%w(http https))] ? home_page : "http://#{home_page}" - elsif request_email_domain - "http://www.#{request_email_domain}" - end - end - # The "internal admin" is a special body for internal use. def self.internal_admin_body matching_pbs = AlaveteliLocalization. @@ -497,7 +487,7 @@ def self.import_csv_from_file(csv_filename, tag, tag_behaviour, dry_run, editor, field_names = { 'name' => 1, 'request_email' => 2 } line = 0 - import_options = {field_names: field_names, + import_options = { field_names: field_names, available_locales: available_locales, tag: tag, tag_behaviour: tag_behaviour, @@ -693,6 +683,7 @@ def json_for_api home_page: calculated_home_page, notes: notes_as_string, publication_scheme: publication_scheme.to_s, + disclosure_log: disclosure_log.to_s, tags: tag_array, info: { requests_count: info_requests_count, @@ -736,7 +727,7 @@ def self.get_request_totals(n, highest, minimum_requests) 'public_bodies' => public_bodies, 'y_values' => y_values, 'y_max' => y_values.max, - 'totals' => y_values} + 'totals' => y_values } end # Return data for the 'n' public bodies with the highest (or @@ -785,7 +776,7 @@ def self.get_request_percentages(column, n, highest, minimum_requests) 'cis_below' => cis_below, 'cis_above' => cis_above, 'y_max' => 100, - 'totals' => original_totals} + 'totals' => original_totals } end def self.popular_bodies(locale) @@ -894,7 +885,7 @@ def update_counter_cache described_state: 'not_held' }, info_requests_successful_count: { awaiting_description: false, described_state: success_states }, - info_requests_visible_classified_count: { awaiting_description: false}, + info_requests_visible_classified_count: { awaiting_description: false }, info_requests_visible_count: {} } @@ -911,6 +902,14 @@ def questions PublicBodyQuestion.fetch(self) end + def cached_urls + [ + public_body_path(self), + list_public_bodies_path, + '^/body/list' + ] + end + private # If the url_name has changed, then all requested_from: queries will break @@ -919,6 +918,10 @@ def reindex_requested_from expire_requests if saved_change_to_attribute?(:url_name) end + def invalidate_cached_pages + NotifyCacheJob.perform_later(self) + end + # Read an attribute value (without using locale fallbacks if the # attribute is translated) def read_attribute_value(name, locale) diff --git a/app/models/public_body/calculated_home_page.rb b/app/models/public_body/calculated_home_page.rb new file mode 100644 index 0000000000..afedc45c63 --- /dev/null +++ b/app/models/public_body/calculated_home_page.rb @@ -0,0 +1,55 @@ +# Guess the home page based on the request email domain. +module PublicBody::CalculatedHomePage + extend ActiveSupport::Concern + + included do + cattr_accessor :excluded_calculated_home_page_domains, default: %w[ + aol.com + gmail.com + googlemail.com + gmx.com + hotmail.com + icloud.com + live.com + mac.com + mail.com + mail.ru + me.com + outlook.com + protonmail.com + qq.com + yahoo.com + yandex.com + ymail.com + zoho.com + ] + end + + def calculated_home_page + @calculated_home_page ||= calculated_home_page! + end + + private + + # Ensure known home page has a full URL or guess if not known. + def calculated_home_page! + ensure_home_page_protocol || guess_home_page + end + + # Ensure the home page has the HTTP protocol at the start of the URL + def ensure_home_page_protocol + return unless home_page.present? + home_page[URI.regexp(%w(http https))] ? home_page : "https://#{home_page}" + end + + # Guess the home page from the request address email domain. + def guess_home_page + return unless request_email_domain + return if excluded_calculated_home_page_domain?(request_email_domain) + "https://www.#{request_email_domain}" + end + + def excluded_calculated_home_page_domain?(domain) + excluded_calculated_home_page_domains.include?(domain) + end +end diff --git a/app/models/user.rb b/app/models/user.rb index 96edfc6434..526b893cb4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -45,21 +45,26 @@ class User < ApplicationRecord include User::Authentication include User::LoginToken include User::OneTimePassword + include User::Slug include User::Survey + include Rails.application.routes.url_helpers + include LinkToHelper - CONTENT_LIMIT = { + DEFAULT_CONTENT_LIMITS = { info_requests: AlaveteliConfiguration.max_requests_per_user_per_day, comments: AlaveteliConfiguration.max_requests_per_user_per_day, user_messages: AlaveteliConfiguration.max_requests_per_user_per_day }.freeze + cattr_accessor :content_limits, default: DEFAULT_CONTENT_LIMITS + rolify before_add: :setup_pro_account, after_add: :assign_role_features, after_remove: :assign_role_features strip_attributes allow_empty: true admin_columns include: [:user_messages_count], - exclude: [:otp_secret_key] + exclude: [:otp_secret_key, :url_name] attr_accessor :no_xapian_reindex @@ -73,6 +78,9 @@ class User < ApplicationRecord has_many :embargoes, inverse_of: :user, through: :info_requests + has_many :outgoing_messages, + inverse_of: :user, + through: :info_requests has_many :draft_info_requests, -> { order(created_at: :desc) }, inverse_of: :user, @@ -173,7 +181,9 @@ class User < ApplicationRecord validate :email_and_name_are_valid after_initialize :set_defaults - after_update :reindex_referencing_models, :update_pro_account + after_update :reindex_referencing_models, + :update_pro_account, + :invalidate_cached_pages acts_as_xapian texts: [:name, :about_me], values: [ @@ -335,6 +345,10 @@ def expire_comments comments.find_each(&:reindex_request_events) end + def invalidate_cached_pages + NotifyCacheJob.perform_later(self) + end + def locale (super || AlaveteliLocalization.locale).to_s end @@ -350,20 +364,17 @@ def name # When name is changed, also change the url name def name=(name) write_attribute(:name, name.try(:strip)) - update_url_name - end - - def update_url_name - url_name = MySociety::Format.simplify_url_part(read_attribute(:name), 'user', 32) - # For user with same name as others, add on arbitary numeric identifier - unique_url_name = url_name - suffix_num = 2 # as there's already one without numeric suffix - conditions = id ? ["id <> ?", id] : [] - until User.where(url_name: unique_url_name).where(conditions).first.nil? - unique_url_name = url_name + "_" + suffix_num.to_s - suffix_num += 1 - end - self.url_name = unique_url_name + end + + def previous_names + outgoing_messages.unscope(:order). + distinct(:from_name). + where.not(from_name: read_attribute(:name)). + pluck(:from_name) + end + + def safe_previous_names + outgoing_messages.map(&:safe_from_name).uniq - [read_attribute(:name)] end # For use in to/from in email messages @@ -431,9 +442,14 @@ def erase! sha = Digest::SHA1.hexdigest(rand.to_s) transaction do + slugs.destroy_all sign_ins.destroy_all profile_photo&.destroy! + outgoing_messages.update!( + from_name: _('[Name Removed]') + ) + update!( name: _('[Name Removed]'), email: "#{sha}@invalid", @@ -445,12 +461,15 @@ def erase! end def anonymise! - return if info_requests.none? - - censor_rules.create!(text: read_attribute(:name), - replacement: _('[Name Removed]'), - last_edit_editor: 'User#anonymise!', - last_edit_comment: 'User#anonymise!') + return if info_requests.none? && comments.none? + + current_name = read_attribute(:name) + [current_name, *previous_names].each do |name| + censor_rules.create!(text: name, + replacement: _('[Name Removed]'), + last_edit_editor: 'User#anonymise!', + last_edit_comment: 'User#anonymise!') + end end def close_and_anonymise @@ -662,6 +681,12 @@ def flipper_id "User;#{id}" end + def cached_urls + [ + user_path(self) + ] + end + private def set_defaults @@ -700,6 +725,6 @@ def update_pro_account end def content_limit(content) - CONTENT_LIMIT[content] + content_limits[content] end end diff --git a/app/models/user/slug.rb b/app/models/user/slug.rb new file mode 100644 index 0000000000..926e1ae4c0 --- /dev/null +++ b/app/models/user/slug.rb @@ -0,0 +1,59 @@ +## +# Module to configure Friendly ID slugs for the User model +# +module User::Slug + extend ActiveSupport::Concern + + included do + extend FriendlyId + + friendly_id do |config| + config.base = :name + config.use :slugged + config.use :sequentially_slugged + config.use :history + + config.slug_column = :url_name + config.sequence_separator = '_' + config.slug_limit = 32 + end + + def should_generate_new_friendly_id? + return true unless url_name + !url_name_changed? && name_changed? && active? + end + + def normalize_friendly_id(_value) + value = read_attribute(:name) + return super('user') if value =~ /^[\d_\.]+$/ + super(value).gsub('-', '_') + end + + def to_param + id + end + + # These private methods reverts from the `history` modules implementation + # to the `slugged` version. This is so generating and searching for slugs + # works correctly as current slugs need to be migrated to `FriendlyId::Slug` + # instances. + private + + # rubocop:disable all + def slug_base_class + self.class.base_class + end + + def slug_column + friendly_id_config.slug_column + end + + def scope_for_slug_generator + scope = self.class.base_class.unscoped + scope = scope.friendly unless scope.respond_to?(:exists_by_friendly_id?) + primary_key_name = self.class.primary_key + scope.where(self.class.base_class.arel_table[primary_key_name].not_eq(send(primary_key_name))) + end + # rubocop:enable all + end +end diff --git a/app/models/user/with_request.rb b/app/models/user/with_request.rb index 615446d339..b08672b5a4 100644 --- a/app/models/user/with_request.rb +++ b/app/models/user/with_request.rb @@ -1,7 +1,7 @@ class User::WithRequest < SimpleDelegator attr_reader :request - delegate :user_agent, to: :request + delegate :ip, :user_agent, to: :request def initialize(user, request) @request = request diff --git a/app/models/webhook.rb b/app/models/webhook.rb index c4e05c2c26..9eb86a647c 100644 --- a/app/models/webhook.rb +++ b/app/models/webhook.rb @@ -37,7 +37,9 @@ def state private def subscription_state - if previous['canceled_at'].nil? && object['canceled_at'] + if previous['status'] == 'incomplete' && object['status'] == 'active' + _('Subscription started') + elsif previous['canceled_at'].nil? && object['canceled_at'] _('Subscription cancelled') elsif previous['canceled_at'] && object['canceled_at'].nil? _('Subscription reactivated') diff --git a/app/views/admin/blog_posts/_about.html.erb b/app/views/admin/blog_posts/_about.html.erb index 00e8965925..bcd5c534e0 100644 --- a/app/views/admin/blog_posts/_about.html.erb +++ b/app/views/admin/blog_posts/_about.html.erb @@ -6,7 +6,8 @@

Alaveteli automatically imports posts from the configured - BLOG_FEED to display around the site. + BLOG_FEED to display around the site. The feed is imported + when the <%= link_to 'public blog page', blog_path %> is viewed.

diff --git a/app/views/admin/changelog/index.html.erb b/app/views/admin/changelog/index.html.erb new file mode 100644 index 0000000000..f1024f385f --- /dev/null +++ b/app/views/admin/changelog/index.html.erb @@ -0,0 +1,22 @@ +<% @title = 'Alaveteli Changelog' %> + +

<%= @title %>

+ +

+ The changelog documents the features and upgrade notes for each versioned + release of Alaveteli. Here you will see the changelog for the version of + Alaveteli you have installed. You can check GitHub for + newer releases of + Alaveteli and preview upcoming changes in the + + development version of the changelog. +

+ +

+ You are currently running Alaveteli version: + <%= link_to ALAVETELI_VERSION, admin_debug_path %> +

+ +
+ +<%= @changelog %> diff --git a/app/views/admin/notes/_show.html.erb b/app/views/admin/notes/_show.html.erb index 5eccb94753..ea5ad9cbef 100644 --- a/app/views/admin/notes/_show.html.erb +++ b/app/views/admin/notes/_show.html.erb @@ -4,6 +4,7 @@ ID Notable ID + Note Body Notable type Notable tag Actions @@ -13,6 +14,7 @@ <%= h note.id %> <%= h note.notable_id %> + <%= truncate(h(note.body), length: 50) %> <%= h note.notable_type %> <%= h note.notable_tag %> <%= link_to "Edit", edit_admin_note_path(note) %> diff --git a/app/views/admin_announcements/_form.html.erb b/app/views/admin_announcements/_form.html.erb index 32c512d82f..f41c59b247 100644 --- a/app/views/admin_announcements/_form.html.erb +++ b/app/views/admin_announcements/_form.html.erb @@ -36,7 +36,7 @@
<% if feature_enabled?(:alaveteli_pro) %>