Merge develop 2024-07-18 (#256)

.gitignore

@@ -6,7 +6,7 @@
 /test/instance
 /test/uploads
 /.elixir_ls
-/test/fixtures/DSCN0010_tmp.jpg
+/test/fixtures/DSCN0010_tmp*
 /test/fixtures/test_tmp.txt
 /test/fixtures/image_tmp.jpg
 /test/tmp/
@@ -60,3 +60,6 @@ pleroma.iml
 *~
 *#
 *.swp
+
+archive-*
+.gitlab-ci-local

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
+image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25
 
 variables: &global_variables
   # Only used for the release
@@ -18,9 +18,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 cache: &global_cache_policy
-  key:
-    files:
-      - mix.lock
+  key: $CI_JOB_IMAGE-$CI_COMMIT_SHORT_SHA
   paths:
     - deps
     - _build
@@ -72,21 +70,20 @@ check-changelog:
   tags:
     - amd64
 
-build-1.13.4:
+build-1.13.4-otp-25:
   extends:
   - .build_changes_policy
   - .using-ci-base
   stage: build
   script:
   - mix compile --force
 
-build-1.15.7-otp-25:
+build-1.17.1-otp-26:
   extends:
   - .build_changes_policy
   - .using-ci-base
   stage: build
-  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
-  allow_failure: true
+  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.17.1-otp-26
   script:
   - mix compile --force
 
@@ -121,7 +118,7 @@ benchmark:
     - mix ecto.migrate
     - mix pleroma.load_testing
 
-unit-testing-1.12.3:
+unit-testing-1.13.4-otp-25:
   extends:
   - .build_changes_policy
   - .using-ci-base
@@ -136,42 +133,27 @@ unit-testing-1.12.3:
   script: &testing_script
     - mix ecto.create
     - mix ecto.migrate
-    - mix test --cover --preload-modules
+    - mix pleroma.test_runner --cover --preload-modules
   coverage: '/^Line total: ([^ ]*%)$/'
   artifacts:
     reports:
       coverage_report:
         coverage_format: cobertura
         path: coverage.xml
 
-unit-testing-1.15.7-otp-25:
+unit-testing-1.17.1-otp-26:
   extends:
   - .build_changes_policy
   - .using-ci-base
   stage: test
-  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
-  allow_failure: true
+  image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.17.1-otp-26
   cache: *testing_cache_policy
   services: *testing_services
   script: *testing_script
 
-unit-testing-1.12-erratic:
-  extends:
-  - .build_changes_policy
-  - .using-ci-base
-  stage: test
-  retry: 2
-  allow_failure: true
-  cache: *testing_cache_policy
-  services: *testing_services
-  script:
-    - mix ecto.create
-    - mix ecto.migrate
-    - mix test --only=erratic
-
-formatting-1.13:
+formatting-1.15:
   extends: .build_changes_policy
-  image: &formatting_elixir elixir:1.13-alpine
+  image: &formatting_elixir elixir:1.15-alpine
   stage: lint
   cache: *testing_cache_policy
   before_script: &current_bfr_script
@@ -183,7 +165,7 @@ formatting-1.13:
   script:
     - mix format --check-formatted
 
-cycles-1.13:
+cycles-1.15:
   extends: .build_changes_policy
   image: *formatting_elixir
   stage: lint

changelog.d/3280-fix-emoji-ids.fix

@@ -0,0 +1 @@
+Fix Emoji object IDs not always being valid

changelog.d/4167-strip-gps-info-in-png.fix

@@ -0,0 +1 @@
+Ensure that StripLocation actually removes everything resembling GPS data from PNGs

changelog.d/adminfe-logger.change

@@ -0,0 +1 @@
+Elixir Logger configuration is now longer permitted through AdminFE and ConfigDB

changelog.d/bandit_update_1.5.2.change

@@ -0,0 +1 @@
+Update Bandit to 1.5.2

changelog.d/cowboy-stream-chunked.fix

@@ -0,0 +1 @@
+Restore Cowboy's ability to stream MediaProxy responses without Chunked encoding.

changelog.d/deps-bump-2024-06-07.skip

@@ -0,0 +1,2 @@
+Update dependencies held back due to old Elixir version
+

changelog.d/docs-netbsd-update.change

@@ -0,0 +1 @@
+Update and extend NetBSD installation docs

changelog.d/elixir-1.15.fix

@@ -0,0 +1 @@
+Elixir 1.15 compatibility

changelog.d/fix-mrfs.add

@@ -0,0 +1 @@
+Added a Mix task "pleroma.config fix_mrf_policies" which will remove erroneous MRF policies from ConfigDB.

changelog.d/group-repeats.fix

@@ -0,0 +1 @@
+Deactivated groups would still try to repeat a post.

changelog.d/gun_pool4.fix

@@ -0,0 +1 @@
+Gun Connection Pool was not retrying to acquire a connection if the pool was full and stale connections were reclaimed

changelog.d/ipfs-dialyzer.skip

@@ -0,0 +1 @@
+no comment

changelog.d/ldap-error-logging.change

@@ -0,0 +1 @@
+Improve error logging when LDAP authentication fails.

changelog.d/ldap.fix

@@ -0,0 +1 @@
+Fix LDAP support

changelog.d/missing-fks.add

@@ -0,0 +1 @@
+Add missing indexes on foreign key relationships

changelog.d/mrf-nsfw-otp25.skip

@@ -0,0 +1 @@
+noop

changelog.d/oban-cancel.change

@@ -0,0 +1 @@
+Changed some jobs to return :cancel on unrecoverable errors that should not be retried

changelog.d/oban-fetcher-rejected.change

@@ -0,0 +1 @@
+Discard Remote Fetcher jobs which errored due to an MRF rejection.

changelog.d/oban-live_dashboard.add

@@ -0,0 +1 @@
+Oban jobs can now be viewed in the Live Dashboard

changelog.d/oban-rich-media-errors.fix

@@ -0,0 +1 @@
+Prevent Rich Media backfill jobs from retrying in cases where it is likely they will fail again.

changelog.d/oban-timeouts.change

@@ -0,0 +1 @@
+Ensure all Oban jobs have timeouts defined

changelog.d/oban-user-refresh-unique.fix

@@ -0,0 +1 @@
+Oban Jobs for refreshing users were not respecting the uniqueness setting

changelog.d/pinned-collection-fetch.security

@@ -0,0 +1 @@
+Use proper workers for fetching pins instead of an ad-hoc task, fixing a potential fetch loop

changelog.d/rich_media_backfill.change

@@ -0,0 +1 @@
+Rich Media backfilling is now an Oban job

changelog.d/stream-end-poll.fix

@@ -0,0 +1 @@
+End of poll notifications were not streamed over websockets or web push

changelog.d/user-refresh.change

@@ -0,0 +1 @@
+User profile refreshes are now asynchronous

changelog.d/webpush-polls.change

@@ -0,0 +1 @@
+Render nice web push notifications for polls

ci/elixir-1.13/Dockerfile → ci/elixir-1.13.4-otp-25/Dockerfile

@@ -1,4 +1,4 @@
-FROM elixir:1.13.4-otp-24
+FROM elixir:1.13.4-otp-25
 
 # Single RUN statement, otherwise intermediate images are created
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run

ci/elixir-1.13/build_and_push.sh → ci/elixir-1.13.4-otp-25/build_and_push.sh

@@ -1 +1 @@
-docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-25 --push .

ci/elixir-1.15-otp25/Dockerfile → ci/elixir-1.15.8-otp-26/Dockerfile

@@ -1,4 +1,4 @@
-FROM elixir:1.15.7-otp-25
+FROM elixir:1.15.8-otp-26
 
 # Single RUN statement, otherwise intermediate images are created
 # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run

ci/elixir-1.15.8-otp-26/build_and_push.sh

@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15.8-otp-26 --push .

ci/elixir-1.16.3-otp-26/Dockerfile

@@ -0,0 +1,8 @@
+FROM elixir:1.16.3-otp-26
+
+# Single RUN statement, otherwise intermediate images are created
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
+RUN apt-get update &&\
+	apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
+	mix local.hex --force &&\
+	mix local.rebar --force

ci/elixir-1.16.3-otp-26/build_and_push.sh

@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.16.3-otp-26 --push .

ci/elixir-1.17.1-otp-26/Dockerfile

@@ -0,0 +1,8 @@
+FROM elixir:1.17.1-otp-26
+
+# Single RUN statement, otherwise intermediate images are created
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
+RUN apt-get update &&\
+	apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
+	mix local.hex --force &&\
+	mix local.rebar --force

ci/elixir-1.17.1-otp-26/build_and_push.sh

@@ -0,0 +1 @@
+docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.17.1-otp-26 --push .

config/config.exs

@@ -83,8 +83,8 @@ config :ex_aws, :s3,
   scheme: "https://"
 
 config :pleroma, Pleroma.Uploaders.IPFS,
-  post_gateway_url: nil,
-  get_gateway_url: nil
+  post_gateway_url: "http://localhost:5001",
+  get_gateway_url: "http://localhost:8080"
 
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
@@ -132,6 +132,8 @@ config :pleroma, Pleroma.Web.Endpoint,
   ]
 
 # Configures Elixir's Logger
+config :logger, backends: [:console]
+
 config :logger, :console,
   level: :debug,
   format: "\n$time $metadata[$level] $message\n",
@@ -585,12 +587,11 @@ config :pleroma, Oban,
     activity_expiration: 10,
     federator_incoming: 5,
     federator_outgoing: 5,
-    ingestion_queue: 50,
     web_push: 50,
     transmogrifier: 20,
-    background: 5,
+    background: 20,
     search_indexing: [limit: 10, paused: true],
-    slow: 1
+    slow: 5
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [

config/description.exs

@@ -1228,79 +1228,6 @@ config :pleroma, :config_description, [
       }
     ]
   },
-  %{
-    group: :logger,
-    type: :group,
-    description: "Logger-related settings",
-    children: [
-      %{
-        key: :backends,
-        type: [:atom, :tuple, :module],
-        description:
-          "Where logs will be sent, :console - send logs to stdout, { ExSyslogger, :ex_syslogger } - to syslog, Quack.Logger - to Slack.",
-        suggestions: [:console, {ExSyslogger, :ex_syslogger}]
-      }
-    ]
-  },
-  %{
-    group: :logger,
-    type: :group,
-    key: :ex_syslogger,
-    label: "ExSyslogger",
-    description: "ExSyslogger-related settings",
-    children: [
-      %{
-        key: :level,
-        type: {:dropdown, :atom},
-        description: "Log level",
-        suggestions: [:debug, :info, :warning, :error]
-      },
-      %{
-        key: :ident,
-        type: :string,
-        description:
-          "A string that's prepended to every message, and is typically set to the app name",
-        suggestions: ["pleroma"]
-      },
-      %{
-        key: :format,
-        type: :string,
-        description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
-        suggestions: ["$metadata[$level] $message"]
-      },
-      %{
-        key: :metadata,
-        type: {:list, :atom},
-        suggestions: [:request_id]
-      }
-    ]
-  },
-  %{
-    group: :logger,
-    type: :group,
-    key: :console,
-    label: "Console Logger",
-    description: "Console logger settings",
-    children: [
-      %{
-        key: :level,
-        type: {:dropdown, :atom},
-        description: "Log level",
-        suggestions: [:debug, :info, :warning, :error]
-      },
-      %{
-        key: :format,
-        type: :string,
-        description: "Default: \"$date $time [$level] $levelpad$node $metadata $message\"",
-        suggestions: ["$metadata[$level] $message"]
-      },
-      %{
-        key: :metadata,
-        type: {:list, :atom},
-        suggestions: [:request_id]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: :frontend_configurations,

config/dev.exs

@@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local
 # different ports.
 
 # Do not include timestamps in development logs
-config :logger, :console, format: "$metadata[$level] $message\n"
+config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n"
 
 # Set a higher stacktrace during development. Avoid configuring such
 # in production as building large stacktraces may be expensive.

config/prod.exs

@@ -20,6 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint,
 config :phoenix, serve_endpoints: true
 
 # Do not print debug messages in production
+config :logger, Logger.Backends.Console, level: :info
 config :logger, :console, level: :info
 config :logger, :ex_syslogger, level: :info
 

config/test.exs

@@ -49,7 +49,8 @@ config :pleroma, Pleroma.Repo,
   hostname: System.get_env("DB_HOST") || "localhost",
   port: System.get_env("DB_PORT") || "5432",
   pool: Ecto.Adapters.SQL.Sandbox,
-  pool_size: System.schedulers_online() * 2
+  pool_size: System.schedulers_online() * 2,
+  log: false
 
 config :pleroma, :dangerzone, override_repo_pool_size: true
 
@@ -177,11 +178,14 @@ config :pleroma, Pleroma.Application,
   streamer_registry: false,
   test_http_pools: true
 
+config :pleroma, Pleroma.Web.Streaming, sync_streaming: true
+
 config :pleroma, Pleroma.Uploaders.Uploader, timeout: 1_000
 
 config :pleroma, Pleroma.Emoji.Loader, test_emoji: true
 
-config :pleroma, Pleroma.Web.RichMedia.Backfill, provider: Pleroma.Web.RichMedia.Backfill
+config :pleroma, Pleroma.Web.RichMedia.Backfill,
+  stream_out: Pleroma.Web.ActivityPub.ActivityPubMock
 
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"

docs/administration/CLI_tasks/config.md

@@ -154,4 +154,19 @@ This forcibly removes all saved values in the database.
 
     ```sh
     mix pleroma.config [--force] reset
+
+    ```
+
+## Remove invalid MRF modules from the database
+
+This forcibly removes any enabled MRF that does not exist and will fix the ability of the instance to start.
+
+=== "OTP"
+    ```sh
+    ./bin/pleroma_ctl config fix_mrf_policies
     ```
+
+=== "From Source"
+    ```sh
+    mix pleroma.config fix_mrf_policies
+    ```