MG-2456 - Refactor auth (#2409)

- Set/Unset parent Group for Things and Channels (#2486) - Move groups out of pkg (#2493) - Separate Things authn and Channels authz (#2496) Signed-off-by: Arvindh <[email protected]> NOISSUE - Add Publish/Subscribe to channels (#2497) Signed-off-by: Arvindh <[email protected]> MG-2457 - Update auth tests (#2503) Signed-off-by: Felix Gateru <[email protected]> MG-2477 - Replace Things with Clients (#2508) Signed-off-by: Dusan Borovcanin <[email protected]> NOISSUE - Rename Things to Clients Signed-off-by: Felix Gateru <[email protected]> Signed-off-by: Arvindh <[email protected]>
absmach · Nov 27, 2024 · 6681cc0 · 6681cc0
1 parent bb5f21b
commit 6681cc0
Show file tree

Hide file tree

Showing 508 changed files with 44,210 additions and 28,511 deletions.
diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml
@@ -19,7 +19,7 @@ on:
       - "journal/api/**"
       - "provision/api/**"
       - "readers/api/**"
-      - "things/api/**"
+      - "clients/api/**"
       - "users/api/**"
 
 env:
@@ -29,7 +29,7 @@ env:
   USER_SECRET: 12345678
   DOMAIN_NAME: demo-test
   USERS_URL: http://localhost:9002
-  THINGS_URL: http://localhost:9000
+  CLIENTS_URL: http://localhost:9000
   HTTP_ADAPTER_URL: http://localhost:8008
   INVITATIONS_URL: http://localhost:9020
   AUTH_URL: http://localhost:8189
@@ -65,8 +65,8 @@ jobs:
           export DOMAIN_ID=$(curl -sSX POST $DOMAINS_URL -H "Content-Type: application/json" -H "Authorization: Bearer $USER_TOKEN" -d "{\"name\":\"$DOMAIN_NAME\",\"alias\":\"$DOMAIN_NAME\"}" | jq -r .id)
           export USER_TOKEN=$(curl -sSX POST $TOKENS_URL -H "Content-Type: application/json" -d "{\"identity\": \"$USER_IDENTITY\",\"secret\": \"$USER_SECRET\",\"domain_id\": \"$DOMAIN_ID\"}" | jq -r .access_token)
           echo "USER_TOKEN=$USER_TOKEN" >> $GITHUB_ENV
-          export THING_SECRET=$(magistrala-cli provision test | /usr/bin/grep -Eo '"secret": "[^"]+"' | awk 'NR % 2 == 0' | sed 's/"secret": "\(.*\)"/\1/')
-          echo "THING_SECRET=$THING_SECRET" >> $GITHUB_ENV
+          export CLIENT_SECRET=$(magistrala-cli provision test | /usr/bin/grep -Eo '"secret": "[^"]+"' | awk 'NR % 2 == 0' | sed 's/"secret": "\(.*\)"/\1/')
+          echo "CLIENT_SECRET=$CLIENT_SECRET" >> $GITHUB_ENV
 
       - name: Check for changes in specific paths
         uses: dorny/paths-filter@v3
@@ -113,10 +113,10 @@ jobs:
               - "api/openapi/readers.yml"
               - "readers/api/**"
 
-            things:
+            clients:
               - ".github/workflows/api-tests.yml"
-              - "api/openapi/things.yml"
-              - "things/api/**"
+              - "api/openapi/clients.yml"
+              - "clients/api/**"
 
             users:
               - ".github/workflows/api-tests.yml"
@@ -133,12 +133,12 @@ jobs:
           report: false
           args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
 
-      - name: Run Things API tests
-        if: steps.changes.outputs.things == 'true'
+      - name: Run Clients API tests
+        if: steps.changes.outputs.clients == 'true'
         uses: schemathesis/action@v1
         with:
-          schema: api/openapi/things.yml
-          base-url: ${{ env.THINGS_URL }}
+          schema: api/openapi/clients.yml
+          base-url: ${{ env.CLIENTS_URL }}
           checks: all
           report: false
           args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
@@ -151,7 +151,7 @@ jobs:
           base-url: ${{ env.HTTP_ADAPTER_URL }}
           checks: all
           report: false
-          args: '--header "Authorization: Thing ${{ env.THING_SECRET }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
+          args: '--header "Authorization: Client ${{ env.CLIENT_SECRET }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
 
       - name: Run Invitations API tests
         if: steps.changes.outputs.invitations == 'true'

diff --git a/.github/workflows/check-generated-files.yml b/.github/workflows/check-generated-files.yml
@@ -49,8 +49,8 @@ jobs:
               - "users/clients.go"
               - "pkg/clients/clients.go"
               - "pkg/messaging/pubsub.go"
-              - "things/postgres/clients.go"
-              - "things/things.go"
+              - "clients/postgres/clients.go"
+              - "clients/clients.go"
               - "pkg/authz.go"
               - "pkg/authn.go"
               - "auth/domains.go"
@@ -132,9 +132,9 @@ jobs:
           mv ./users/mocks/repository.go ./users/mocks/repository.go.tmp
           mv ./users/mocks/service.go ./users/mocks/service.go.tmp
           mv ./pkg/messaging/mocks/pubsub.go ./pkg/messaging/mocks/pubsub.go.tmp
-          mv ./things/mocks/repository.go ./things/mocks/repository.go.tmp
-          mv ./things/mocks/service.go ./things/mocks/service.go.tmp
-          mv ./things/mocks/cache.go ./things/mocks/cache.go.tmp
+          mv ./clients/mocks/repository.go ./clients/mocks/repository.go.tmp
+          mv ./clients/mocks/service.go ./clients/mocks/service.go.tmp
+          mv ./clients/mocks/cache.go ./clients/mocks/cache.go.tmp
           mv ./auth/mocks/authz.go ./auth/mocks/authz.go.tmp
           mv ./auth/mocks/domains.go ./auth/mocks/domains.go.tmp
           mv ./auth/mocks/keys.go ./auth/mocks/keys.go.tmp
@@ -161,7 +161,7 @@ jobs:
           mv ./journal/mocks/repository.go ./journal/mocks/repository.go.tmp
           mv ./journal/mocks/service.go ./journal/mocks/service.go.tmp
           mv ./auth/mocks/domains_client.go ./auth/mocks/domains_client.go.tmp
-          mv ./things/mocks/things_client.go ./things/mocks/things_client.go.tmp
+          mv ./clients/mocks/clients_client.go ./clients/mocks/clients_client.go.tmp
           mv ./pkg/authz/mocks/authz.go ./pkg/authz/mocks/authz.go.tmp
           mv ./pkg/authn/mocks/authn.go ./pkg/authn/mocks/authn.go.tmp
 
@@ -183,9 +183,9 @@ jobs:
           check_mock_changes ./users/mocks/repository.go "Users Repository ./users/mocks/repository.go"
           check_mock_changes ./users/mocks/service.go "Users Service ./users/mocks/service.go"
           check_mock_changes ./pkg/messaging/mocks/pubsub.go "PubSub ./pkg/messaging/mocks/pubsub.go"
-          check_mock_changes ./things/mocks/repository.go "Things Repository ./things/mocks/repository.go"
-          check_mock_changes ./things/mocks/service.go "Things Service ./things/mocks/service.go"
-          check_mock_changes ./things/mocks/cache.go "Things Cache ./things/mocks/cache.go"
+          check_mock_changes ./clients/mocks/repository.go "Clients Repository ./clients/mocks/repository.go"
+          check_mock_changes ./clients/mocks/service.go "Clients Service ./clients/mocks/service.go"
+          check_mock_changes ./clients/mocks/cache.go "Clients Cache ./clients/mocks/cache.go"
           check_mock_changes ./auth/mocks/authz.go "Auth Authz ./auth/mocks/authz.go"
           check_mock_changes ./auth/mocks/domains.go "Auth Domains ./auth/mocks/domains.go"
           check_mock_changes ./auth/mocks/keys.go "Auth Keys ./auth/mocks/keys.go"
@@ -214,4 +214,4 @@ jobs:
           check_mock_changes ./journal/mocks/service.go "Journal Service ./journal/mocks/service.go"
           check_mock_changes ./auth/mocks/domains_client.go "Domains Service Client ./auth/mocks/domains_client.go"
           check_mock_changes ./auth/mocks/token_client.go "Token Service Client ./auth/mocks/token_client.go"
-          check_mock_changes ./things/mocks/things_client.go "Things Service Client things/mocks/things_client.go"
+          check_mock_changes .clients/mocks/clients_client.go "Clients Service Client clients/mocks/clients_client.go"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -111,7 +111,7 @@ jobs:
               - "cmd/coap/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
 
             consumers:
@@ -140,7 +140,7 @@ jobs:
               - "cmd/http/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
               - "logger/**"
 
@@ -163,7 +163,7 @@ jobs:
               - "cmd/mqtt/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
               - "logger/**"
               - "pkg/events/**"
@@ -197,7 +197,7 @@ jobs:
               - "invitations/**"
               - "provision/**"
               - "readers/**"
-              - "things/**"
+              - "clients/**"
               - "users/**"
 
             pkg-transformers:
@@ -221,12 +221,12 @@ jobs:
               - "cmd/timescale-reader/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "auth/**"
 
-            things:
-              - "things/**"
-              - "cmd/things/**"
+            clients:
+              - "clients/**"
+              - "cmd/clients/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
               - "auth/**"
@@ -249,7 +249,7 @@ jobs:
               - "cmd/ws/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
 
       - name: Create coverage directory
@@ -366,10 +366,10 @@ jobs:
         run: |
           go test --race -v -count=1 -coverprofile=coverage/readers.out ./readers/...
 
-      - name: Run things tests
-        if: steps.changes.outputs.things == 'true' || steps.changes.outputs.workflow == 'true'
+      - name: Run clients tests
+        if: steps.changes.outputs.clients == 'true' || steps.changes.outputs.workflow == 'true'
         run: |
-          go test --race -v -count=1 -coverprofile=coverage/things.out ./things/...
+          go test --race -v -count=1 -coverprofile=coverage/clients.out ./clients/...
 
       - name: Run users tests
         if: steps.changes.outputs.users == 'true' || steps.changes.outputs.workflow == 'true'

diff --git a/Makefile b/Makefile
@@ -2,10 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 MG_DOCKER_IMAGE_NAME_PREFIX ?= magistrala
-BUILD_DIR = build
-SERVICES = auth users things http coap ws postgres-writer postgres-reader timescale-writer \
+BUILD_DIR ?= build
+SERVICES = auth users clients groups channels domains http coap ws postgres-writer postgres-reader timescale-writer \
 	timescale-reader cli bootstrap mqtt provision certs invitations journal
-TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers things users
+TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers clients users
 TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES))
 DOCKERS = $(addprefix docker_,$(SERVICES))
 DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES))
@@ -19,10 +19,14 @@ empty:=
 space:= $(empty) $(empty)
 # Docker compose project name should follow this guidelines: https://docs.docker.com/compose/reference/#use--p-to-specify-a-project-name
 DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)) | tr -c -s '[:alnum:][=-=]' '_' | tr '[:upper:]' '[:lower:]')
-DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
+DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config restart
 DEFAULT_DOCKER_COMPOSE_COMMAND  := up
 GRPC_MTLS_CERT_FILES_EXISTS = 0
 MOCKERY_VERSION=v2.43.2
+INTERNAL_PROTO_GEN_OUT_DIR=internal/grpc
+INTERNAL_PROTO_DIR=internal/proto
+INTERNAL_PROTO_FILES := $(shell find $(INTERNAL_PROTO_DIR) -name "*.proto" | sed 's|$(INTERNAL_PROTO_DIR)/||')
+
 ifneq ($(MG_MESSAGE_BROKER_TYPE),)
     MG_MESSAGE_BROKER_TYPE := $(MG_MESSAGE_BROKER_TYPE)
 else
@@ -138,8 +142,8 @@ define test_api_service
 		exit 1; \
 	fi
 
-	@if [ "$(svc)" = "http" ] && [ -z "$(THING_SECRET)" ]; then \
-		echo "THING_SECRET is not set"; \
+	@if [ "$(svc)" = "http" ] && [ -z "$(CLIENT_SECRET)" ]; then \
+		echo "CLIENT_SECRET is not set"; \
 		echo "Please set it to a valid secret"; \
 		exit 1; \
 	fi
@@ -148,7 +152,7 @@ define test_api_service
 		st run api/openapi/$(svc).yml \
 		--checks all \
 		--base-url $(2) \
-		--header "Authorization: Thing $(THING_SECRET)" \
+		--header "Authorization: Client $(CLIENT_SECRET)" \
 		--contrib-openapi-formats-uuid \
 		--hypothesis-suppress-health-check=filter_too_much \
 		--stateful=links; \
@@ -164,7 +168,7 @@ define test_api_service
 endef
 
 test_api_users: TEST_API_URL := http://localhost:9002
-test_api_things: TEST_API_URL := http://localhost:9000
+test_api_clients: TEST_API_URL := http://localhost:9000
 test_api_http: TEST_API_URL := http://localhost:8008
 test_api_invitations: TEST_API_URL := http://localhost:9020
 test_api_auth: TEST_API_URL := http://localhost:8189
@@ -179,7 +183,8 @@ $(TEST_API):
 
 proto:
 	protoc -I. --go_out=. --go_opt=paths=source_relative pkg/messaging/*.proto
-	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative ./*.proto
+	mkdir -p $(INTERNAL_PROTO_GEN_OUT_DIR)
+	protoc -I $(INTERNAL_PROTO_DIR) --go_out=$(INTERNAL_PROTO_GEN_OUT_DIR) --go_opt=paths=source_relative --go-grpc_out=$(INTERNAL_PROTO_GEN_OUT_DIR) --go-grpc_opt=paths=source_relative $(INTERNAL_PROTO_FILES)
 
 $(FILTERED_SERVICES):
 	$(call compile_service,$(@))
@@ -218,7 +223,7 @@ rundev:
 	cd scripts && ./run.sh
 
 grpc_mtls_certs:
-	$(MAKE) -C docker/ssl auth_grpc_certs things_grpc_certs
+	$(MAKE) -C docker/ssl auth_grpc_certs clients_grpc_certs
 
 check_tls:
 ifeq ($(GRPC_TLS),true)
@@ -244,7 +249,7 @@ check_certs: check_mtls check_tls
 ifeq ($(GRPC_MTLS_CERT_FILES_EXISTS),0)
 ifeq ($(filter true,$(GRPC_MTLS) $(GRPC_TLS)),true)
 ifeq ($(filter $(DEFAULT_DOCKER_COMPOSE_COMMAND),$(DOCKER_COMPOSE_COMMAND)),$(DEFAULT_DOCKER_COMPOSE_COMMAND))
-	$(MAKE) -C docker/ssl auth_grpc_certs things_grpc_certs
+	$(MAKE) -C docker/ssl auth_grpc_certs clients_grpc_certs
 endif
 endif
 endif
@@ -257,3 +262,6 @@ run_addons: check_certs
 	@for SVC in $(RUN_ADDON_ARGS); do \
 		MG_ADDONS_CERTS_PATH_PREFIX="../."  docker compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \
 	done
+
+run_live: check_certs
+	GOPATH=$(go env GOPATH) docker compose  -f docker/docker-compose.yml -f docker/docker-compose-live.yaml   --env-file docker/.env -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
diff --git a/README.md b/README.md
@@ -137,7 +137,8 @@ You like SuperMQ and you would like to make it your day job? We're always lookin
 
 [Apache-2.0](LICENSE)
 
-[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fabsmach%2Fmagistrala.svg?type=large&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2Fabsmach%2Fmagistrala?ref=badge_large&issueType=license)
+[![FOSSA Status][FOSSA]][FOSSA]
+
 ## Data Collection for SuperMQ
 
 SuperMQ is committed to continuously improving its services and ensuring a seamless experience for its users. To achieve this, we collect certain data from your deployments. Rest assured, this data is collected solely for the purpose of enhancing SuperMQ and is not used with any malicious intent. The deployment summary can be found on our [website][callhome].
@@ -189,3 +190,4 @@ By utilizing SuperMQ, you actively contribute to its improvement. Together, we c
 [rodneyosodo]: https://github.com/rodneyosodo
 [callhome]: https://deployments.magistrala.abstractmachines.fr/
 [contrib]: https://www.github.com/absmach/mg-contrib
+[FOSSA]: https://app.fossa.com/api/projects/git%2Bgithub.com%2Fabsmach%2Fsupermq.svg?type=small
diff --git a/api/asyncapi/mqtt.yml b/api/asyncapi/mqtt.yml
@@ -17,8 +17,8 @@ info:
   license:
     name: Apache 2.0
     url: 'https://github.com/absmach/magistrala/blob/main/LICENSE'
-  
-  
+
+
 defaultContentType: application/json
 
 servers:
@@ -33,7 +33,7 @@ servers:
         enum:
           - '1883'
           - '8883'
-    security: 
+    security:
       - user-password: []
 
 channels:
@@ -45,7 +45,7 @@ channels:
         required: true
       subtopic:
         $ref: '#/components/parameters/subtopic'
-        in: path 
+        in: path
         required: false
 
     publish:
@@ -88,7 +88,7 @@ components:
 
   parameters:
     channelID:
-      description: Channel ID connected to the Thing ID defined in the username.
+      description: Channel ID connected to the Client ID defined in the username.
       schema:
         type: string
         format: uuid
@@ -97,13 +97,13 @@ components:
       schema:
         type: string
         default: ''
-  
+
   securitySchemes:
     user-password:
       type: userPassword
       description: |
-        username is thing ID connected to the channel defined in the mqtt topic and 
-        password is thing key corresponding to the thing ID 
+        username is client ID connected to the channel defined in the mqtt topic and
+        password is client secret corresponding to the client ID
 
   operationTraits:
     mqtt:

diff --git a/api/asyncapi/websocket.yml b/api/asyncapi/websocket.yml
@@ -126,7 +126,7 @@ components:
         ```
   parameters:
     channelID:
-      description: Channel ID connected to the Thing ID defined in the username.
+      description: Channel ID connected to the Client ID defined in the username.
       schema:
         type: string
         format: uuid
@@ -141,4 +141,4 @@ components:
       scheme: bearer
       bearerFormat: uuid
       description: |
-        * Thing access: "Authorization: Thing <thing_key>"
+        * Client access: "Authorization: Client <client_key>"
diff --git a/api/openapi/auth.yml b/api/openapi/auth.yml
@@ -31,7 +31,7 @@ tags:
     externalDocs:
       description: Find out more about domains
       url: https://docs.magistrala.abstractmachines.fr/
-  
+
   - name: Health
     description: Service health check endpoint.
     externalDocs:
@@ -549,7 +549,7 @@ components:
         metadata:
           type: object
           example: { "domain": "example.com" }
-          description: Arbitrary, object-encoded thing's data.
+          description: Arbitrary, object-encoded client's data.
         alias:
           type: string
           example: domain alias