Merge pull request #25 from newrelic/feat/org-limits

feat: add support for reporting org limits; refactored to be more generic... again

CONTRIBUTING.md

@@ -1,10 +1,10 @@
 # Contributing
 
 Contributions are always welcome. Before contributing please read the
-[code of conduct](./CODE_OF_CONDUCT.md) and [search the issue tracker](issues); your issue may have already been discussed or fixed in `main`. To contribute,
+[code of conduct](https://github.com/newrelic/.github/blob/main/CODE_OF_CONDUCT.md) and [search the issue tracker](issues); your issue may have already been discussed or fixed in `main`. To contribute,
 [fork](https://help.github.com/articles/fork-a-repo/) this repository, commit your changes, and [send a Pull Request](https://help.github.com/articles/using-pull-requests/).
 
-Note that our [code of conduct](./CODE_OF_CONDUCT.md) applies to all platforms and venues related to this project; please follow it in all your interactions with the project and its participants.
+Note that our [code of conduct](https://github.com/newrelic/.github/blob/main/CODE_OF_CONDUCT.md) applies to all platforms and venues related to this project; please follow it in all your interactions with the project and its participants.
 
 ## Feature Requests
 
@@ -27,4 +27,4 @@ For more information about CLAs, please check out Alex Russell’s excellent pos
 
 ## Slack
 
-We host a public Slack with a dedicated channel for contributors and maintainers of open source projects hosted by New Relic.  If you are contributing to this project, you're welcome to request access to the #oss-contributors channel in the newrelicusers.slack.com workspace.  To request access, see https://newrelicusers-signup.herokuapp.com/.
+We host a public Slack with a dedicated channel for contributors and maintainers of open source projects hosted by New Relic.  If you are contributing to this project, you're welcome to request access to the #oss-contributors channel in the newrelicusers.slack.com workspace.  To request access, please use this [link](https://join.slack.com/t/newrelicusers/shared_invite/zt-1ayj69rzm-~go~Eo1whIQGYnu3qi15ng).

config_sample.yml

@@ -1,58 +1,57 @@
-integration_name: com.newrelic.labs.sfdc.eventlogfiles
-run_as_service: False
-cron_interval_minutes: 60
-service_schedule: { "hour":  "*", "minute": "0,15,30,45" }
-instances:
-  - name: sfdc-logs
-    arguments:
-      api_ver: "55.0"
-      token_url: "https://test.salesforce.com/services/oauth2/token"
-      auth: {
-        "grant_type": "password",
-        "client_id": "",
-        "client_secret": "",
-        "username": "",
-        "password": ""
-      }
-      auth_env_prefix: "MY_"
-      cache_enabled: False
-      redis: {
-        "host": "",
-        "port": "",
-        "db_number": 0,
-        "password": "",
-        "ssl": True,
-        "expire_days": 2
-      }
-      date_field: "LogDate"
-      generation_interval: "Hourly"
-      time_lag_minutes: 300
-    labels:
-      environment: production
-queries: [
-  {
-    query: "SELECT Id,EventType,CreatedDate,LogDate,LogFile,Interval FROM EventLogFile WHERE CreatedDate>={from_timestamp} AND EventType='API' AND Interval='{log_interval_type}'"
-  },
-  {
-    query: "SELECT Id,Action,CreatedDate,DelegateUser,Display FROM SetupAuditTrail WHERE CreatedDate>={from_timestamp}",
-    timestamp_attr: CreatedData,
-    rename_timestamp: actualTimestamp,
-    api_ver: "58.0"
-  },
-  {
-    query: "SELECT EventName, EventType, UsageType, Client, Value, StartDate, EndDate FROM PlatformEventUsageMetric WHERE TimeSegment='FifteenMinutes' AND StartDate >= {start_date} AND EndDate <= {end_date}",
-    env: {
-      end_date: "now()",
-      start_date: "now(timedelta(minutes=-60))"
-    },
-    api_ver: "58.0",
-    timestamp_attr: StartDate,
-  },
-  "other_queries.yml"
-]
-newrelic:
-  data_format: "events"
-  api_endpoint: "US"
-  account_id: ""
-  license_key: ""
-
+integration_name: com.newrelic.labs.sfdc.eventlogfiles
+run_as_service: False
+cron_interval_minutes: 60
+service_schedule:
+  hour: *
+  minute: "0,15,30,45"
+instances:
+  - name: sfdc-logs
+    arguments:
+      api_ver: "55.0"
+      token_url: "https://test.salesforce.com/services/oauth2/token"
+      auth:
+        grant_type: password
+        client_id: "ABCDEFG"
+        client_secret: "ABCD1234"
+        username: salesforce_user
+        password: "MY_SALESFORCE_PASSWORD"
+      auth_env_prefix: MY_
+      cache_enabled: False
+      redis:
+        host: my.redis.test
+        port: 6379
+        db_number: 0
+        password: "MY_REDIS_PASSWORD"
+        ssl: True
+        expire_days: 2
+      date_field: LogDate
+      generation_interval: Hourly
+      time_lag_minutes: 300
+      queries:
+      - query: SELECT * FROM Account
+      - query: SELECT * FROM SetupAuditTrail
+      limits:
+        api_ver: "55.0"
+        names:
+        - ActiveScratchOrgs
+        - DailyApiRequests
+      logs_enabled: yes
+    labels:
+      environment: production
+queries:
+- query: "SELECT Id,EventType,CreatedDate,LogDate,LogFile,Interval FROM EventLogFile WHERE CreatedDate>={from_timestamp} AND EventType='API' AND Interval='{log_interval_type}'"
+- query: "SELECT Id,Action,CreatedDate,DelegateUser,Display FROM SetupAuditTrail WHERE CreatedDate>={from_timestamp}"
+  timestamp_attr: CreatedDate
+  rename_timestamp: actualTimestamp
+  api_ver: "58.0"
+- query: "SELECT EventName, EventType, UsageType, Client, Value, StartDate, EndDate FROM PlatformEventUsageMetric WHERE TimeSegment='FifteenMinutes' AND StartDate >= {start_date} AND EndDate <= {end_date}"
+  env:
+    end_date: "now()"
+    start_date: "now(timedelta(minutes=-60))"
+  api_ver: "58.0"
+  timestamp_attr: StartDate
+newrelic:
+  data_format: events
+  api_endpoint: US
+  account_id: "MY_NEW_RELIC_ACCOUNT_ID"
+  license_key: "MY_NEW_RELIC_LICENSE_KEY"

requirements.txt

@@ -16,3 +16,4 @@ wheel==0.38.1
 setuptools==65.5.1
 future~=0.18.2
 newrelic==9.7.0
+coverage~=7.4

src/__main__.py

@@ -2,27 +2,24 @@
 import newrelic.agent
 newrelic.agent.initialize('./newrelic.ini')
 
+
 import optparse
 import os
+from pytz import utc
 import sys
 from typing import Any
+from yaml import Loader, load
 
 
 from apscheduler.executors.pool import ThreadPoolExecutor
 from apscheduler.jobstores.memory import MemoryJobStore
 from apscheduler.schedulers.background import BlockingScheduler
-from pytz import utc
-from yaml import Loader, load
-from newrelic_logging.api import ApiFactory
-from newrelic_logging.auth import AuthenticatorFactory
-from newrelic_logging.cache import CacheFactory, BackendFactory
-from newrelic_logging.config import Config, getenv
-from newrelic_logging.newrelic import NewRelicFactory
-from newrelic_logging.pipeline import PipelineFactory
-from newrelic_logging.query import QueryFactory
-from newrelic_logging.salesforce import SalesForceFactory
 
-from newrelic_logging.integration import Integration
+
+from newrelic_logging.config import Config, getenv
+from newrelic_logging.factory import Factory
+from newrelic_logging.limits import receiver as limits_receiver
+from newrelic_logging.query import QueryFactory, receiver as query_receiver
 from newrelic_logging.telemetry import print_info, print_warn
 
 
@@ -130,30 +127,48 @@ def load_mapping_file(mapping_file_path: str, default_mapping: Any) -> dict:
         return load(stream, Loader=Loader)[MAPPING]
 
 
-def run_once(
+def create_receivers(
     config: Config,
     event_type_fields_mapping: dict,
-    numeric_fields_list: set
+    initial_delay: int,
 ):
+    receivers = []
+
+    receivers.append(
+        query_receiver.new_create_receiver_func(
+            config,
+            QueryFactory(),
+            event_type_fields_mapping,
+            initial_delay,
+        )
+    )
+
+    receivers.append(
+        limits_receiver.new_create_receiver_func()
+    )
+
+    return receivers
 
-    Integration(
+
+def run_once(
+    factory: Factory,
+    config: Config,
+    receivers: list[callable],
+    numeric_fields_list: set
+):
+    # Run the integration
+    factory.new_integration(
+        factory,
         config,
-        AuthenticatorFactory(),
-        CacheFactory(BackendFactory()),
-        PipelineFactory(),
-        SalesForceFactory(),
-        ApiFactory(),
-        QueryFactory(),
-        NewRelicFactory(),
-        event_type_fields_mapping,
+        receivers,
         numeric_fields_list,
-        config.get_int(CRON_INTERVAL_MINUTES, 60),
     ).run()
 
 
 def run_as_service(
+    factory: Factory,
     config: Config,
-    event_type_fields_mapping: dict,
+    receivers: list[callable],
     numeric_fields_list: set,
 ):
     scheduler = BlockingScheduler(
@@ -171,17 +186,11 @@ def run_as_service(
 
     service_schedule = config[SERVICE_SCHEDULE]
     scheduler.add_job(
-        Integration(
+        factory.new_integration(
+            factory,
             config,
-            AuthenticatorFactory(),
-            CacheFactory(BackendFactory()),
-            PipelineFactory(),
-            SalesForceFactory(),
-            QueryFactory(),
-            NewRelicFactory(),
-            event_type_fields_mapping,
+            receivers,
             numeric_fields_list,
-            0
         ).run,
         trigger='cron',
         hour=service_schedule['hour'],
@@ -198,11 +207,33 @@ def run(
     event_type_fields_mapping: dict,
     numeric_fields_list: set
 ):
+    factory = Factory()
+
     if not config.get(RUN_AS_SERVICE, False):
-        run_once(config, event_type_fields_mapping, numeric_fields_list)
+        run_once(
+            factory,
+            config,
+            create_receivers(
+                config,
+                event_type_fields_mapping,
+                config.get_int(CRON_INTERVAL_MINUTES, 60),
+            ),
+            numeric_fields_list,
+        )
         return
 
-    run_as_service(config, event_type_fields_mapping, numeric_fields_list)
+
+    run_as_service(
+        factory,
+        config,
+        create_receivers(
+            config,
+            event_type_fields_mapping,
+            0,
+        ),
+        numeric_fields_list,
+    )
+
 
 @newrelic.agent.background_task()
 def main():

src/newrelic_logging/api.py

@@ -85,6 +85,8 @@ def query(self, session: Session, soql: str, api_ver: str = None) -> dict:
         if not api_ver is None:
             ver = api_ver
 
+        # @TODO handle pagination
+
         return get(
             self.authenticator,
             session,
@@ -106,9 +108,14 @@ def get_log_file(
             stream=True,
         )
 
-class ApiFactory:
-    def __init__(self):
-        pass
+    def list_limits(self, session: Session, api_ver: str = None) -> dict:
+        ver = self.api_ver
+        if not api_ver is None:
+            ver = api_ver
 
-    def new(self, authenticator: Authenticator, api_ver: str) -> Api:
-        return Api(authenticator, api_ver)
+        return get(
+            self.authenticator,
+            session,
+            f'/services/data/v{ver}/limits/',
+            lambda response : response.json(),
+        )

src/newrelic_logging/auth.py

@@ -213,7 +213,7 @@ def validate_oauth_config(auth: dict) -> dict:
         raise ConfigException('username', 'missing OAuth username')
 
     if not auth['password']:
-        raise ConfigException('password', 'missing OAuth client secret')
+        raise ConfigException('password', 'missing OAuth password')
 
     return auth
 
@@ -287,27 +287,3 @@ def make_auth_from_env(config: Config) -> dict:
         })
 
     raise Exception(f'Wrong or missing grant_type')
-
-
-class AuthenticatorFactory:
-    def __init__(self):
-        pass
-
-    def new(self, config: Config, data_cache: DataCache) -> Authenticator:
-        token_url = config.get('token_url', env_var_name=SF_TOKEN_URL)
-
-        if not token_url:
-            raise ConfigException('token_url', 'missing token URL')
-
-        if 'auth' in config:
-            return Authenticator(
-                token_url,
-                make_auth_from_config(config.sub('auth')),
-                data_cache,
-            )
-
-        return Authenticator(
-            token_url,
-            make_auth_from_env(config),
-            data_cache
-        )