From 05535b1193a82ac6caef875eef0b201a69bad4a4 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 4 Oct 2024 11:25:14 +0200 Subject: [PATCH] Force DatetimeUTC everywhere (#456) --- .github/workflows/python_ci.yaml | 6 +- .../match_bets_with_langfuse_traces.py | 8 +- poetry.lock | 21 +++-- .../benchmark/agents.py | 12 +-- .../deploy/agent.py | 8 +- .../deploy/gcp/kubernetes_models.py | 7 +- prediction_market_agent_tooling/gtypes.py | 5 +- .../jobs/jobs_models.py | 6 +- .../jobs/omen/omen_jobs.py | 4 +- .../markets/agent_market.py | 28 +++---- .../markets/data_models.py | 6 +- .../markets/manifold/api.py | 12 +-- .../markets/manifold/data_models.py | 36 +++------ .../markets/manifold/manifold.py | 4 +- .../markets/markets.py | 10 ++- .../markets/metaculus/api.py | 5 +- .../markets/metaculus/data_models.py | 21 ++--- .../markets/metaculus/metaculus.py | 4 +- .../markets/omen/data_models.py | 57 ++++++++----- .../markets/omen/omen.py | 19 +++-- .../markets/omen/omen_contracts.py | 5 +- .../markets/omen/omen_resolving.py | 5 +- .../markets/omen/omen_subgraph_handler.py | 69 ++++++++-------- .../markets/polymarket/data_models.py | 11 ++- .../markets/polymarket/data_models_web.py | 28 +++---- .../markets/polymarket/polymarket.py | 4 +- .../monitor/markets/manifold.py | 4 +- .../monitor/markets/metaculus.py | 4 +- .../monitor/markets/omen.py | 4 +- .../monitor/markets/polymarket.py | 4 +- .../monitor/monitor.py | 20 ++--- .../monitor/monitor_app.py | 15 ++-- .../tools/contract.py | 10 +-- .../tools/datetime_utc.py | 74 +++++++++++++++++ .../tools/langfuse_client_utils.py | 27 ++++--- .../tools/tavily_storage/tavily_models.py | 8 +- .../tools/utils.py | 80 +++++++------------ pyproject.toml | 6 +- scripts/create_market_omen.py | 5 +- tests/markets/omen/test_omen.py | 23 +++--- .../omen/test_omen_subgraph_handler.py | 14 ++-- tests/markets/test_manifold.py | 8 +- tests/markets/test_markets.py | 9 +++ tests/tools/test_datetime_utc.py | 52 ++++++++++++ 44 files changed, 442 insertions(+), 326 deletions(-) create mode 100644 prediction_market_agent_tooling/tools/datetime_utc.py create mode 100644 tests/tools/test_datetime_utc.py diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index 4e9fb11c..f3206906 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -36,11 +36,11 @@ jobs: python-version: [ '3.10.x', '3.11.x', '3.12.x' ] test: - name: Unit Tests - command: 'poetry run python -m pytest tests/ -p no:ape_test' + command: 'poetry run python -m pytest tests/ -p no:ape_test -vvv' - name: Integration Tests - command: 'poetry run python -m pytest tests_integration/ -p no:ape_test' + command: 'poetry run python -m pytest tests_integration/ -p no:ape_test -vvv' - name: Integration with Local Chain - command: 'poetry run python -m pytest tests_integration_with_local_chain/ --disable-isolation' + command: 'poetry run python -m pytest tests_integration_with_local_chain/ --disable-isolation -vvv' name: pytest - Python ${{ matrix.python-version }} - ${{ matrix.test.name }} steps: - uses: actions/checkout@v2 diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 1942f634..ded3b8a0 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -1,4 +1,3 @@ -from datetime import datetime from typing import Any import pandas as pd @@ -24,7 +23,10 @@ get_trace_for_bet, get_traces_for_agent, ) -from prediction_market_agent_tooling.tools.utils import get_private_key_from_gcp_secret +from prediction_market_agent_tooling.tools.utils import ( + get_private_key_from_gcp_secret, + utc_datetime, +) class SimulatedOutcome(BaseModel): @@ -160,7 +162,7 @@ def get_outcome_for_trace( api_keys = APIKeys(BET_FROM_PRIVATE_KEY=private_key) # Pick a time after pool token number is stored in OmenAgentMarket - start_time = datetime(2024, 9, 13) + start_time = utc_datetime(2024, 9, 13) langfuse = Langfuse( secret_key=api_keys.langfuse_secret_key.get_secret_value(), diff --git a/poetry.lock b/poetry.lock index c4c999c5..4250d50b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -5058,13 +5058,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlmodel" -version = "0.0.21" +version = "0.0.22" description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." optional = false python-versions = ">=3.7" files = [ - {file = "sqlmodel-0.0.21-py3-none-any.whl", hash = "sha256:eca104afe8a643f0764076b29f02e51d19d6b35c458f4c119942960362a4b52a"}, - {file = "sqlmodel-0.0.21.tar.gz", hash = "sha256:b2034c23d930f66d2091b17a4280a9c23a7ea540a71e7fcf9c746d262f06f74a"}, + {file = "sqlmodel-0.0.22-py3-none-any.whl", hash = "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b"}, + {file = "sqlmodel-0.0.22.tar.gz", hash = "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e"}, ] [package.dependencies] @@ -5384,6 +5384,17 @@ rich = ">=10.11.0" shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" +[[package]] +name = "types-python-dateutil" +version = "2.9.0.20241003" +description = "Typing stubs for python-dateutil" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-python-dateutil-2.9.0.20241003.tar.gz", hash = "sha256:58cb85449b2a56d6684e41aeefb4c4280631246a0da1a719bdbe6f3fb0317446"}, + {file = "types_python_dateutil-2.9.0.20241003-py3-none-any.whl", hash = "sha256:250e1d8e80e7bbc3a6c99b907762711d1a1cdd00e978ad39cb5940f6f0a87f3d"}, +] + [[package]] name = "types-pytz" version = "2024.2.0.20241003" @@ -5888,4 +5899,4 @@ openai = ["openai"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "4f60565b99290a6a4d57f0dafb5523548c01f5d10315f7ec396cc5db0018c8b2" +content-hash = "3894cc80ab3eb8ed82dfdf1a3582d197ffbc3831a15e7fce7fad266e0541ef21" diff --git a/prediction_market_agent_tooling/benchmark/agents.py b/prediction_market_agent_tooling/benchmark/agents.py index 46760ff9..ef093118 100644 --- a/prediction_market_agent_tooling/benchmark/agents.py +++ b/prediction_market_agent_tooling/benchmark/agents.py @@ -1,12 +1,12 @@ import random import typing as t -from datetime import datetime from prediction_market_agent_tooling.benchmark.utils import ( OutcomePrediction, Prediction, ) from prediction_market_agent_tooling.gtypes import Probability +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class AbstractBenchmarkedAgent: @@ -41,7 +41,7 @@ def check_and_predict(self, market_question: str) -> Prediction: def is_predictable_restricted( self, market_question: str, - time_restriction_up_to: datetime, + time_restriction_up_to: DatetimeUTC, ) -> bool: """ Override if the agent can decide to not predict the question, before doing the hard work. @@ -53,7 +53,7 @@ def is_predictable_restricted( def predict_restricted( self, market_question: str, - time_restriction_up_to: datetime, + time_restriction_up_to: DatetimeUTC, ) -> Prediction: """ Predict the outcome of the market question. @@ -65,7 +65,7 @@ def predict_restricted( def check_and_predict_restricted( self, market_question: str, - time_restriction_up_to: datetime, + time_restriction_up_to: DatetimeUTC, ) -> Prediction: """ Data used must be restricted to the time_restriction_up_to. @@ -94,7 +94,7 @@ def predict(self, market_question: str) -> Prediction: ) def predict_restricted( - self, market_question: str, time_restriction_up_to: datetime + self, market_question: str, time_restriction_up_to: DatetimeUTC ) -> Prediction: return self.predict(market_question) @@ -117,6 +117,6 @@ def predict(self, market_question: str) -> Prediction: ) def predict_restricted( - self, market_question: str, time_restriction_up_to: datetime + self, market_question: str, time_restriction_up_to: DatetimeUTC ) -> Prediction: return self.predict(market_question) diff --git a/prediction_market_agent_tooling/deploy/agent.py b/prediction_market_agent_tooling/deploy/agent.py index 82cf5556..a3b941b4 100644 --- a/prediction_market_agent_tooling/deploy/agent.py +++ b/prediction_market_agent_tooling/deploy/agent.py @@ -4,7 +4,7 @@ import tempfile import time import typing as t -from datetime import datetime, timedelta +from datetime import timedelta from enum import Enum from functools import cached_property @@ -67,7 +67,7 @@ from prediction_market_agent_tooling.tools.ipfs.ipfs_handler import IPFSHandler from prediction_market_agent_tooling.tools.is_predictable import is_predictable_binary from prediction_market_agent_tooling.tools.langfuse_ import langfuse_context, observe -from prediction_market_agent_tooling.tools.utils import DatetimeWithTimezone, utcnow +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, utcnow from prediction_market_agent_tooling.tools.web3_utils import ipfscidv0_to_byte32 MAX_AVAILABLE_MARKETS = 20 @@ -215,7 +215,7 @@ def deploy_gcp( secrets: dict[str, str] | None = None, cron_schedule: str | None = None, gcp_fname: str | None = None, - start_time: DatetimeWithTimezone | None = None, + start_time: DatetimeUTC | None = None, timeout: int = 180, ) -> None: path_to_agent_file = os.path.relpath(inspect.getfile(self.__class__)) @@ -287,7 +287,7 @@ def run(self, market_type: MarketType) -> None: raise NotImplementedError("This method must be implemented by the subclass.") def get_gcloud_fname(self, market_type: MarketType) -> str: - return f"{self.__class__.__name__.lower()}-{market_type}-{datetime.now().strftime('%Y-%m-%d--%H-%M-%S')}" + return f"{self.__class__.__name__.lower()}-{market_type}-{utcnow().strftime('%Y-%m-%d--%H-%M-%S')}" class DeployableTraderAgent(DeployableAgent): diff --git a/prediction_market_agent_tooling/deploy/gcp/kubernetes_models.py b/prediction_market_agent_tooling/deploy/gcp/kubernetes_models.py index f8cec5fc..41d4b284 100644 --- a/prediction_market_agent_tooling/deploy/gcp/kubernetes_models.py +++ b/prediction_market_agent_tooling/deploy/gcp/kubernetes_models.py @@ -1,11 +1,10 @@ -from datetime import datetime from typing import Any from pydantic import BaseModel class Metadata(BaseModel): - creationTimestamp: datetime + creationTimestamp: int generation: int name: str namespace: str @@ -15,12 +14,12 @@ class Metadata(BaseModel): class Metadata1(BaseModel): - creationTimestamp: datetime | None + creationTimestamp: int | None name: str class Metadata2(BaseModel): - creationTimestamp: datetime | None + creationTimestamp: int | None name: str diff --git a/prediction_market_agent_tooling/gtypes.py b/prediction_market_agent_tooling/gtypes.py index a612b678..49b04985 100644 --- a/prediction_market_agent_tooling/gtypes.py +++ b/prediction_market_agent_tooling/gtypes.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from typing import NewType, Union from eth_typing.evm import ( # noqa: F401 # Import for the sake of easy importing with others from here. @@ -17,6 +16,9 @@ Wei, ) +from prediction_market_agent_tooling.tools.datetime_utc import ( # noqa: F401 # Import for the sake of easy importing with others from here. + DatetimeUTC, +) from prediction_market_agent_tooling.tools.hexbytes_custom import ( # noqa: F401 # Import for the sake of easy importing with others from here. HexBytes, ) @@ -32,7 +34,6 @@ Probability = NewType("Probability", float) Mana = NewType("Mana", float) # Manifold's "currency" USDC = NewType("USDC", float) -DatetimeWithTimezone = NewType("DatetimeWithTimezone", datetime) ChainID = NewType("ChainID", int) IPFSCIDVersion0 = NewType("IPFSCIDVersion0", str) diff --git a/prediction_market_agent_tooling/jobs/jobs_models.py b/prediction_market_agent_tooling/jobs/jobs_models.py index fd316312..653da7b0 100644 --- a/prediction_market_agent_tooling/jobs/jobs_models.py +++ b/prediction_market_agent_tooling/jobs/jobs_models.py @@ -1,6 +1,5 @@ import typing as t from abc import ABC, abstractmethod -from datetime import datetime from pydantic import BaseModel @@ -9,6 +8,7 @@ FilterBy, SortBy, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class SimpleJob(BaseModel): @@ -16,7 +16,7 @@ class SimpleJob(BaseModel): job: str reward: float currency: str - deadline: datetime + deadline: DatetimeUTC class JobAgentMarket(AgentMarket, ABC): @@ -29,7 +29,7 @@ def job(self) -> str: @property @abstractmethod - def deadline(self) -> datetime: + def deadline(self) -> DatetimeUTC: """Deadline for the job completion.""" @abstractmethod diff --git a/prediction_market_agent_tooling/jobs/omen/omen_jobs.py b/prediction_market_agent_tooling/jobs/omen/omen_jobs.py index 9efd014e..ea80e20a 100644 --- a/prediction_market_agent_tooling/jobs/omen/omen_jobs.py +++ b/prediction_market_agent_tooling/jobs/omen/omen_jobs.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from web3 import Web3 @@ -21,6 +20,7 @@ OmenSubgraphHandler, SortBy, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class OmenJobAgentMarket(OmenAgentMarket, JobAgentMarket): @@ -32,7 +32,7 @@ def job(self) -> str: return self.question @property - def deadline(self) -> datetime: + def deadline(self) -> DatetimeUTC: return self.close_time def get_reward(self, max_bond: float) -> float: diff --git a/prediction_market_agent_tooling/markets/agent_market.py b/prediction_market_agent_tooling/markets/agent_market.py index bff0d4be..358f8486 100644 --- a/prediction_market_agent_tooling/markets/agent_market.py +++ b/prediction_market_agent_tooling/markets/agent_market.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from enum import Enum from eth_typing import ChecksumAddress @@ -18,8 +17,8 @@ TokenAmount, ) from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, check_not_none, - convert_to_utc_datetime, should_not_happen, utcnow, ) @@ -52,23 +51,16 @@ class AgentMarket(BaseModel): question: str description: str | None outcomes: list[str] - outcome_token_pool: dict[ - str, float - ] | None # Should be in currency of `currency` above. + outcome_token_pool: ( + dict[str, float] | None + ) # Should be in currency of `currency` above. resolution: Resolution | None - created_time: datetime | None - close_time: datetime | None + created_time: DatetimeUTC | None + close_time: DatetimeUTC | None current_p_yes: Probability url: str volume: float | None # Should be in currency of `currency` above. - _add_timezone_validator_created_time = field_validator("created_time")( - convert_to_utc_datetime - ) - _add_timezone_validator_close_time = field_validator("close_time")( - convert_to_utc_datetime - ) - @field_validator("outcome_token_pool") def validate_outcome_token_pool( cls, @@ -182,7 +174,7 @@ def get_binary_markets( limit: int, sort_by: SortBy, filter_by: FilterBy = FilterBy.OPEN, - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, ) -> t.Sequence["AgentMarket"]: raise NotImplementedError("Subclasses must implement this method") @@ -193,13 +185,15 @@ def get_binary_market(id: str) -> "AgentMarket": @staticmethod def get_bets_made_since( - better_address: ChecksumAddress, start_time: datetime + better_address: ChecksumAddress, start_time: DatetimeUTC ) -> list[Bet]: raise NotImplementedError("Subclasses must implement this method") @staticmethod def get_resolved_bets_made_since( - better_address: ChecksumAddress, start_time: datetime, end_time: datetime | None + better_address: ChecksumAddress, + start_time: DatetimeUTC, + end_time: DatetimeUTC | None, ) -> list[ResolvedBet]: raise NotImplementedError("Subclasses must implement this method") diff --git a/prediction_market_agent_tooling/markets/data_models.py b/prediction_market_agent_tooling/markets/data_models.py index 333a36e1..f542e51b 100644 --- a/prediction_market_agent_tooling/markets/data_models.py +++ b/prediction_market_agent_tooling/markets/data_models.py @@ -1,10 +1,10 @@ -from datetime import datetime from enum import Enum from typing import Annotated, TypeAlias from pydantic import BaseModel, BeforeValidator, computed_field from prediction_market_agent_tooling.gtypes import OutcomeStr, Probability +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class Currency(str, Enum): @@ -40,7 +40,7 @@ class Bet(BaseModel): id: str amount: BetAmount outcome: bool - created_time: datetime + created_time: DatetimeUTC market_question: str market_id: str @@ -50,7 +50,7 @@ def __str__(self) -> str: class ResolvedBet(Bet): market_outcome: bool - resolved_time: datetime + resolved_time: DatetimeUTC profit: ProfitAmount @computed_field # type: ignore[prop-decorator] diff --git a/prediction_market_agent_tooling/markets/manifold/api.py b/prediction_market_agent_tooling/markets/manifold/api.py index 8c4d847e..61aac3b5 100644 --- a/prediction_market_agent_tooling/markets/manifold/api.py +++ b/prediction_market_agent_tooling/markets/manifold/api.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime import requests import tenacity @@ -20,6 +19,7 @@ ) from prediction_market_agent_tooling.tools.parallelism import par_map from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, response_list_to_model, response_to_model, ) @@ -47,7 +47,7 @@ def get_manifold_binary_markets( ] | None ) = "open", - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, ) -> list[ManifoldMarket]: all_markets: list[ManifoldMarket] = [] @@ -167,8 +167,8 @@ def get_manifold_market(market_id: str) -> FullManifoldMarket: ) def get_manifold_bets( user_id: str, - start_time: datetime, - end_time: t.Optional[datetime], + start_time: DatetimeUTC, + end_time: t.Optional[DatetimeUTC], ) -> list[ManifoldBet]: url = f"{MANIFOLD_API_BASE_URL}/v0/bets" @@ -182,8 +182,8 @@ def get_manifold_bets( def get_resolved_manifold_bets( user_id: str, - start_time: datetime, - end_time: t.Optional[datetime], + start_time: DatetimeUTC, + end_time: t.Optional[DatetimeUTC], ) -> tuple[list[ManifoldBet], list[ManifoldMarket]]: bets = get_manifold_bets(user_id, start_time, end_time) markets: list[ManifoldMarket] = par_map( diff --git a/prediction_market_agent_tooling/markets/manifold/data_models.py b/prediction_market_agent_tooling/markets/manifold/data_models.py index ee0f6ada..0503dc84 100644 --- a/prediction_market_agent_tooling/markets/manifold/data_models.py +++ b/prediction_market_agent_tooling/markets/manifold/data_models.py @@ -1,8 +1,7 @@ import typing as t -from datetime import datetime, timedelta from enum import Enum -from pydantic import BaseModel, field_validator +from pydantic import BaseModel from prediction_market_agent_tooling.gtypes import Mana, Probability from prediction_market_agent_tooling.markets.data_models import ( @@ -10,7 +9,7 @@ ProfitAmount, Resolution, ) -from prediction_market_agent_tooling.tools.utils import should_not_happen +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, should_not_happen MANIFOLD_BASE_URL = "https://manifold.markets" @@ -33,7 +32,7 @@ class ManifoldAnswersMode(str, Enum): class ManifoldAnswer(BaseModel): - createdTime: datetime + createdTime: DatetimeUTC avatarUrl: str id: str username: str @@ -55,17 +54,17 @@ class ManifoldMarket(BaseModel): id: str question: str creatorId: str - closeTime: datetime - createdTime: datetime + closeTime: DatetimeUTC + createdTime: DatetimeUTC creatorAvatarUrl: t.Optional[str] = None creatorName: str creatorUsername: str isResolved: bool resolution: t.Optional[Resolution] = None - resolutionTime: t.Optional[datetime] = None - lastBetTime: t.Optional[datetime] = None - lastCommentTime: t.Optional[datetime] = None - lastUpdatedTime: datetime + resolutionTime: t.Optional[DatetimeUTC] = None + lastBetTime: t.Optional[DatetimeUTC] = None + lastCommentTime: t.Optional[DatetimeUTC] = None + lastUpdatedTime: DatetimeUTC mechanism: str outcomeType: str p: t.Optional[float] = None @@ -100,15 +99,6 @@ def is_resolved_non_cancelled(self) -> bool: def __repr__(self) -> str: return f"Manifold's market: {self.question}" - @field_validator("closeTime", mode="before") - def clip_timestamp(cls, value: int) -> datetime: - """ - Clip the timestamp to the maximum valid timestamp. - """ - max_timestamp = (datetime.max - timedelta(days=1)).timestamp() - value = int(min(value / 1000, max_timestamp)) - return datetime.fromtimestamp(value) - class FullManifoldMarket(ManifoldMarket): # Some of these fields are available only in specific cases, see https://docs.manifold.markets/api#get-v0marketmarketid. @@ -137,7 +127,7 @@ class ManifoldUser(BaseModel): """ id: str - createdTime: datetime + createdTime: DatetimeUTC name: str username: str url: str @@ -154,7 +144,7 @@ class ManifoldUser(BaseModel): userDeleted: t.Optional[bool] = None balance: Mana totalDeposits: Mana - lastBetTime: t.Optional[datetime] = None + lastBetTime: t.Optional[DatetimeUTC] = None currentBettingStreak: t.Optional[int] = None profitCached: ProfitCached @@ -193,7 +183,7 @@ class ManifoldBet(BaseModel): loanAmount: Mana | None orderAmount: t.Optional[Mana] = None fills: t.Optional[list[ManifoldBetFills]] = None - createdTime: datetime + createdTime: DatetimeUTC outcome: Resolution def get_resolved_boolean_outcome(self) -> bool: @@ -237,4 +227,4 @@ class ManifoldContractMetric(BaseModel): userUsername: str userName: str userAvatarUrl: str - lastBetTime: datetime + lastBetTime: DatetimeUTC diff --git a/prediction_market_agent_tooling/markets/manifold/manifold.py b/prediction_market_agent_tooling/markets/manifold/manifold.py index 16e1c873..7cbc2c00 100644 --- a/prediction_market_agent_tooling/markets/manifold/manifold.py +++ b/prediction_market_agent_tooling/markets/manifold/manifold.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from math import ceil from prediction_market_agent_tooling.config import APIKeys @@ -23,6 +22,7 @@ from prediction_market_agent_tooling.tools.betting_strategies.minimum_bet_to_win import ( minimum_bet_to_win, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class ManifoldAgentMarket(AgentMarket): @@ -83,7 +83,7 @@ def get_binary_markets( limit: int, sort_by: SortBy, filter_by: FilterBy = FilterBy.OPEN, - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, ) -> t.Sequence["ManifoldAgentMarket"]: sort: t.Literal["newest", "close-date"] | None diff --git a/prediction_market_agent_tooling/markets/markets.py b/prediction_market_agent_tooling/markets/markets.py index 143571b4..3ef9988b 100644 --- a/prediction_market_agent_tooling/markets/markets.py +++ b/prediction_market_agent_tooling/markets/markets.py @@ -1,5 +1,5 @@ import typing as t -from datetime import datetime, timedelta +from datetime import timedelta from enum import Enum from prediction_market_agent_tooling.config import APIKeys @@ -26,7 +26,11 @@ from prediction_market_agent_tooling.markets.polymarket.polymarket import ( PolymarketAgentMarket, ) -from prediction_market_agent_tooling.tools.utils import should_not_happen, utcnow +from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, + should_not_happen, + utcnow, +) class MarketType(str, Enum): @@ -57,7 +61,7 @@ def get_binary_markets( filter_by: FilterBy = FilterBy.OPEN, sort_by: SortBy = SortBy.NONE, excluded_questions: set[str] | None = None, - created_after: datetime | None = None, + created_after: DatetimeUTC | None = None, ) -> t.Sequence[AgentMarket]: agent_market_class = MARKET_TYPE_TO_AGENT_MARKET[market_type] markets = agent_market_class.get_binary_markets( diff --git a/prediction_market_agent_tooling/markets/metaculus/api.py b/prediction_market_agent_tooling/markets/metaculus/api.py index 08dc0444..0555b2e2 100644 --- a/prediction_market_agent_tooling/markets/metaculus/api.py +++ b/prediction_market_agent_tooling/markets/metaculus/api.py @@ -1,4 +1,3 @@ -from datetime import datetime from typing import Union import requests @@ -9,7 +8,7 @@ MetaculusQuestion, MetaculusQuestions, ) -from prediction_market_agent_tooling.tools.utils import response_to_model +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, response_to_model METACULUS_API_BASE_URL = "https://www.metaculus.com/api2" @@ -65,7 +64,7 @@ def get_questions( order_by: str | None = None, offset: int = 0, tournament_id: int | None = None, - created_after: datetime | None = None, + created_after: DatetimeUTC | None = None, status: str | None = None, ) -> list[MetaculusQuestion]: """ diff --git a/prediction_market_agent_tooling/markets/metaculus/data_models.py b/prediction_market_agent_tooling/markets/metaculus/data_models.py index 11c57408..3204c64d 100644 --- a/prediction_market_agent_tooling/markets/metaculus/data_models.py +++ b/prediction_market_agent_tooling/markets/metaculus/data_models.py @@ -1,9 +1,10 @@ -from datetime import datetime from enum import Enum from typing import Any from pydantic import BaseModel +from prediction_market_agent_tooling.tools.utils import DatetimeUTC + class QuestionType(str, Enum): forecast = "forecast" @@ -33,7 +34,7 @@ def p_yes(self) -> float: class Prediction(BaseModel): - t: datetime + t: DatetimeUTC x: float @@ -68,20 +69,20 @@ class MetaculusQuestion(BaseModel): group_label: str | None = None resolution: int | None resolved_option: int | None - created_time: datetime - publish_time: datetime | None = None - close_time: datetime | None = None - effected_close_time: datetime | None - resolve_time: datetime | None = None + created_time: DatetimeUTC + publish_time: DatetimeUTC | None = None + close_time: DatetimeUTC | None = None + effected_close_time: DatetimeUTC | None + resolve_time: DatetimeUTC | None = None possibilities: dict[Any, Any] | None = None scoring: dict[Any, Any] = {} type: QuestionType | None = None user_perms: Any weekly_movement: float | None weekly_movement_direction: int | None = None - cp_reveal_time: datetime | None = None - edited_time: datetime - last_activity_time: datetime + cp_reveal_time: DatetimeUTC | None = None + edited_time: DatetimeUTC + last_activity_time: DatetimeUTC activity: float comment_count: int votes: int diff --git a/prediction_market_agent_tooling/markets/metaculus/metaculus.py b/prediction_market_agent_tooling/markets/metaculus/metaculus.py index 9d1ab9b1..143758fa 100644 --- a/prediction_market_agent_tooling/markets/metaculus/metaculus.py +++ b/prediction_market_agent_tooling/markets/metaculus/metaculus.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.gtypes import Probability @@ -17,6 +16,7 @@ from prediction_market_agent_tooling.markets.metaculus.data_models import ( MetaculusQuestion, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class MetaculusAgentMarket(AgentMarket): @@ -52,7 +52,7 @@ def get_binary_markets( limit: int, sort_by: SortBy = SortBy.NONE, filter_by: FilterBy = FilterBy.OPEN, - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, tournament_id: int | None = None, ) -> t.Sequence["MetaculusAgentMarket"]: diff --git a/prediction_market_agent_tooling/markets/omen/data_models.py b/prediction_market_agent_tooling/markets/omen/data_models.py index a429085d..84a21379 100644 --- a/prediction_market_agent_tooling/markets/omen/data_models.py +++ b/prediction_market_agent_tooling/markets/omen/data_models.py @@ -1,7 +1,5 @@ import typing as t -from datetime import datetime -import pytz from pydantic import BaseModel, ConfigDict, Field, computed_field from web3 import Web3 @@ -26,6 +24,7 @@ ResolvedBet, ) from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, check_not_none, should_not_happen, ) @@ -71,7 +70,7 @@ class Question(BaseModel): outcomes: list[str] isPendingArbitration: bool openingTimestamp: int - answerFinalizedTimestamp: t.Optional[datetime] = None + answerFinalizedTimestamp: t.Optional[DatetimeUTC] = None currentAnswer: t.Optional[str] = None @property @@ -84,8 +83,8 @@ def n_outcomes(self) -> int: return len(self.outcomes) @property - def opening_datetime(self) -> datetime: - return datetime.fromtimestamp(self.openingTimestamp) + def opening_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.openingTimestamp) @property def has_answer(self) -> bool: @@ -218,11 +217,11 @@ def openingTimestamp(self) -> int: return self.question.openingTimestamp @property - def opening_datetime(self) -> datetime: - return datetime.fromtimestamp(self.openingTimestamp, tz=pytz.UTC) + def opening_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.openingTimestamp) @property - def close_time(self) -> datetime: + def close_time(self) -> DatetimeUTC: # Opening of the Reality's question is close time for the market, # however, market is usually "closed" even sooner by removing all the liquidity. return self.opening_datetime @@ -257,13 +256,13 @@ def question_title(self) -> str: return self.title @property - def creation_datetime(self) -> datetime: - return datetime.fromtimestamp(self.creationTimestamp) + def creation_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.creationTimestamp) @property - def finalized_datetime(self) -> datetime | None: + def finalized_datetime(self) -> DatetimeUTC | None: return ( - datetime.fromtimestamp(self.answerFinalizedTimestamp) + DatetimeUTC.to_datetime_utc(self.answerFinalizedTimestamp) if self.answerFinalizedTimestamp is not None else None ) @@ -490,8 +489,8 @@ class OmenBet(BaseModel): fpmm: OmenMarket @property - def creation_datetime(self) -> datetime: - return datetime.fromtimestamp(self.creationTimestamp, tz=pytz.UTC) + def creation_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.creationTimestamp) @property def boolean_outcome(self) -> bool: @@ -548,9 +547,7 @@ def to_generic_resolved_bet(self) -> ResolvedBet: market_question=self.title, market_id=self.fpmm.id, market_outcome=self.fpmm.boolean_outcome, - resolved_time=datetime.fromtimestamp( - check_not_none(self.fpmm.answerFinalizedTimestamp) - ), + resolved_time=check_not_none(self.fpmm.finalized_datetime), profit=self.get_profit(), ) @@ -571,11 +568,23 @@ class RealityQuestion(BaseModel): id: str user: HexAddress historyHash: HexBytes | None - updatedTimestamp: datetime + updatedTimestamp: int contentHash: HexBytes questionId: HexBytes - answerFinalizedTimestamp: datetime - currentScheduledFinalizationTimestamp: datetime + answerFinalizedTimestamp: int + currentScheduledFinalizationTimestamp: int + + @property + def updated_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.updatedTimestamp) + + @property + def answer_finalized_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.answerFinalizedTimestamp) + + @property + def current_scheduled_finalization_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.currentScheduledFinalizationTimestamp) @property def url(self) -> str: @@ -584,13 +593,17 @@ def url(self) -> str: class RealityAnswer(BaseModel): id: str - timestamp: datetime + timestamp: int answer: HexBytes lastBond: Wei bondAggregate: Wei question: RealityQuestion createdBlock: int + @property + def timestamp_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.timestamp) + class RealityResponse(BaseModel): """ @@ -598,7 +611,7 @@ class RealityResponse(BaseModel): """ id: str - timestamp: datetime + timestamp: int answer: HexBytes isUnrevealed: bool isCommitment: bool diff --git a/prediction_market_agent_tooling/markets/omen/omen.py b/prediction_market_agent_tooling/markets/omen/omen.py index 260ef5d9..d1b88343 100644 --- a/prediction_market_agent_tooling/markets/omen/omen.py +++ b/prediction_market_agent_tooling/markets/omen/omen.py @@ -1,6 +1,6 @@ import sys import typing as t -from datetime import datetime, timedelta +from datetime import timedelta import tenacity from web3 import Web3 @@ -70,6 +70,7 @@ ) from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, calculate_sell_amount_in_collateral, check_not_none, ) @@ -97,9 +98,9 @@ class OmenAgentMarket(AgentMarket): collateral_token_contract_address_checksummed: ChecksumAddress market_maker_contract_address_checksummed: ChecksumAddress condition: Condition - finalized_time: datetime | None - created_time: datetime - close_time: datetime + finalized_time: DatetimeUTC | None + created_time: DatetimeUTC + close_time: DatetimeUTC fee: float # proportion, from 0 to 1 _binary_market_p_yes_history: list[Probability] | None = None @@ -363,7 +364,7 @@ def get_binary_markets( limit: int, sort_by: SortBy, filter_by: FilterBy = FilterBy.OPEN, - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, ) -> t.Sequence["OmenAgentMarket"]: return [ @@ -387,7 +388,7 @@ def get_binary_market(id: str) -> "OmenAgentMarket": @staticmethod def get_bets_made_since( - better_address: ChecksumAddress, start_time: datetime + better_address: ChecksumAddress, start_time: DatetimeUTC ) -> list[Bet]: bets = OmenSubgraphHandler().get_bets( better_address=better_address, start_time=start_time @@ -397,7 +398,9 @@ def get_bets_made_since( @staticmethod def get_resolved_bets_made_since( - better_address: ChecksumAddress, start_time: datetime, end_time: datetime | None + better_address: ChecksumAddress, + start_time: DatetimeUTC, + end_time: DatetimeUTC | None, ) -> list[ResolvedBet]: subgraph_handler = OmenSubgraphHandler() bets = subgraph_handler.get_resolved_bets_with_valid_answer( @@ -836,7 +839,7 @@ def omen_create_market_tx( api_keys: APIKeys, initial_funds: xDai, question: str, - closing_time: datetime, + closing_time: DatetimeUTC, category: str, language: str, outcomes: list[str], diff --git a/prediction_market_agent_tooling/markets/omen/omen_contracts.py b/prediction_market_agent_tooling/markets/omen/omen_contracts.py index 134d8c07..02c8c3d1 100644 --- a/prediction_market_agent_tooling/markets/omen/omen_contracts.py +++ b/prediction_market_agent_tooling/markets/omen/omen_contracts.py @@ -1,7 +1,7 @@ import os import random import typing as t -from datetime import datetime, timedelta +from datetime import timedelta from enum import Enum from web3 import Web3 @@ -40,6 +40,7 @@ init_collateral_token_contract, to_gnosis_chain_contract, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC from prediction_market_agent_tooling.tools.web3_utils import ( ZERO_BYTES, byte32_to_ipfscidv0, @@ -575,7 +576,7 @@ def askQuestion( outcomes: list[str], language: str, arbitrator: Arbitrator, - opening: datetime, + opening: DatetimeUTC, timeout: timedelta, nonce: int | None = None, tx_params: t.Optional[TxParams] = None, diff --git a/prediction_market_agent_tooling/markets/omen/omen_resolving.py b/prediction_market_agent_tooling/markets/omen/omen_resolving.py index 99c03cc6..2851366d 100644 --- a/prediction_market_agent_tooling/markets/omen/omen_resolving.py +++ b/prediction_market_agent_tooling/markets/omen/omen_resolving.py @@ -1,5 +1,3 @@ -from datetime import datetime - from web3 import Web3 from prediction_market_agent_tooling.config import APIKeys @@ -33,6 +31,7 @@ from prediction_market_agent_tooling.markets.polymarket.utils import ( find_resolution_on_polymarket, ) +from prediction_market_agent_tooling.tools.utils import utcnow from prediction_market_agent_tooling.tools.web3_utils import ZERO_BYTES, xdai_to_wei @@ -131,7 +130,7 @@ def finalize_markets( logger.info( f"[{idx+1} / {len(markets_with_resolutions)}] Looking into {market.url=} {market.question_title=}" ) - closed_before_days = (datetime.now() - market.close_time).days + closed_before_days = (utcnow() - market.close_time).days if resolution is None: if closed_before_days > wait_n_days_before_invalid: diff --git a/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py b/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py index 7ee33991..12721d24 100644 --- a/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py +++ b/prediction_market_agent_tooling/markets/omen/omen_subgraph_handler.py @@ -1,6 +1,5 @@ import sys import typing as t -from datetime import datetime import requests import tenacity @@ -35,7 +34,11 @@ sDaiContract, ) from prediction_market_agent_tooling.tools.singleton import SingletonMeta -from prediction_market_agent_tooling.tools.utils import to_int_timestamp, utcnow +from prediction_market_agent_tooling.tools.utils import ( + DatetimeUTC, + to_int_timestamp, + utcnow, +) from prediction_market_agent_tooling.tools.web3_utils import ( ZERO_BYTES, byte32_to_ipfscidv0, @@ -213,11 +216,11 @@ def _build_where_statements( creator: t.Optional[HexAddress] = None, creator_in: t.Optional[t.Sequence[HexAddress]] = None, outcomes: list[str] = OMEN_BINARY_MARKET_OUTCOMES, - created_after: t.Optional[datetime] = None, - opened_before: t.Optional[datetime] = None, - opened_after: t.Optional[datetime] = None, - finalized_before: t.Optional[datetime] = None, - finalized_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, + opened_before: t.Optional[DatetimeUTC] = None, + opened_after: t.Optional[DatetimeUTC] = None, + finalized_before: t.Optional[DatetimeUTC] = None, + finalized_after: t.Optional[DatetimeUTC] = None, finalized: bool | None = None, resolved: bool | None = None, liquidity_bigger_than: Wei | None = None, @@ -344,7 +347,7 @@ def get_omen_binary_markets_simple( filter_by: FilterBy, sort_by: SortBy, # Additional filters, these can not be modified by the enums above. - created_after: datetime | None = None, + created_after: DatetimeUTC | None = None, excluded_questions: set[str] | None = None, # question titles collateral_token_address_in: ( tuple[ChecksumAddress, ...] | None @@ -357,7 +360,7 @@ def get_omen_binary_markets_simple( # These values need to be set according to the filter_by value, so they can not be passed as arguments. finalized: bool | None = None resolved: bool | None = None - opened_after: datetime | None = None + opened_after: DatetimeUTC | None = None liquidity_bigger_than: Wei | None = None if filter_by == FilterBy.RESOLVED: @@ -393,11 +396,11 @@ def get_omen_binary_markets_simple( def get_omen_binary_markets( self, limit: t.Optional[int], - created_after: t.Optional[datetime] = None, - opened_before: t.Optional[datetime] = None, - opened_after: t.Optional[datetime] = None, - finalized_before: t.Optional[datetime] = None, - finalized_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, + opened_before: t.Optional[DatetimeUTC] = None, + opened_after: t.Optional[DatetimeUTC] = None, + finalized_before: t.Optional[DatetimeUTC] = None, + finalized_after: t.Optional[DatetimeUTC] = None, finalized: bool | None = None, resolved: bool | None = None, creator: t.Optional[HexAddress] = None, @@ -554,12 +557,12 @@ def get_user_positions( def get_trades( self, better_address: ChecksumAddress | None = None, - start_time: datetime | None = None, - end_time: t.Optional[datetime] = None, + start_time: DatetimeUTC | None = None, + end_time: t.Optional[DatetimeUTC] = None, market_id: t.Optional[ChecksumAddress] = None, filter_by_answer_finalized_not_null: bool = False, type_: t.Literal["Buy", "Sell"] | None = None, - market_opening_after: datetime | None = None, + market_opening_after: DatetimeUTC | None = None, collateral_amount_more_than: Wei | None = None, ) -> list[OmenBet]: if not end_time: @@ -597,11 +600,11 @@ def get_trades( def get_bets( self, better_address: ChecksumAddress | None = None, - start_time: datetime | None = None, - end_time: t.Optional[datetime] = None, + start_time: DatetimeUTC | None = None, + end_time: t.Optional[DatetimeUTC] = None, market_id: t.Optional[ChecksumAddress] = None, filter_by_answer_finalized_not_null: bool = False, - market_opening_after: datetime | None = None, + market_opening_after: DatetimeUTC | None = None, collateral_amount_more_than: Wei | None = None, ) -> list[OmenBet]: return self.get_trades( @@ -618,8 +621,8 @@ def get_bets( def get_resolved_bets( self, better_address: ChecksumAddress, - start_time: datetime, - end_time: t.Optional[datetime] = None, + start_time: DatetimeUTC, + end_time: t.Optional[DatetimeUTC] = None, market_id: t.Optional[ChecksumAddress] = None, ) -> list[OmenBet]: omen_bets = self.get_bets( @@ -634,8 +637,8 @@ def get_resolved_bets( def get_resolved_bets_with_valid_answer( self, better_address: ChecksumAddress, - start_time: datetime, - end_time: t.Optional[datetime] = None, + start_time: DatetimeUTC, + end_time: t.Optional[DatetimeUTC] = None, market_id: t.Optional[ChecksumAddress] = None, ) -> list[OmenBet]: bets = self.get_resolved_bets( @@ -650,9 +653,9 @@ def get_resolved_bets_with_valid_answer( def get_reality_question_filters( user: HexAddress | None = None, claimed: bool | None = None, - current_answer_before: datetime | None = None, - finalized_before: datetime | None = None, - finalized_after: datetime | None = None, + current_answer_before: DatetimeUTC | None = None, + finalized_before: DatetimeUTC | None = None, + finalized_after: DatetimeUTC | None = None, id_in: list[str] | None = None, question_id: HexBytes | None = None, question_id_in: list[HexBytes] | None = None, @@ -699,9 +702,9 @@ def get_questions( limit: int | None, user: HexAddress | None = None, claimed: bool | None = None, - current_answer_before: datetime | None = None, - finalized_before: datetime | None = None, - finalized_after: datetime | None = None, + current_answer_before: DatetimeUTC | None = None, + finalized_before: DatetimeUTC | None = None, + finalized_after: DatetimeUTC | None = None, id_in: list[str] | None = None, question_id_in: list[HexBytes] | None = None, ) -> list[RealityQuestion]: @@ -744,9 +747,9 @@ def get_responses( user: HexAddress | None = None, question_id: HexBytes | None = None, question_claimed: bool | None = None, - question_finalized_before: datetime | None = None, - question_finalized_after: datetime | None = None, - question_current_answer_before: datetime | None = None, + question_finalized_before: DatetimeUTC | None = None, + question_finalized_after: DatetimeUTC | None = None, + question_current_answer_before: DatetimeUTC | None = None, question_id_in: list[HexBytes] | None = None, ) -> list[RealityResponse]: where_stms: dict[str, t.Any] = {} diff --git a/prediction_market_agent_tooling/markets/polymarket/data_models.py b/prediction_market_agent_tooling/markets/polymarket/data_models.py index 8255a240..f294f7d2 100644 --- a/prediction_market_agent_tooling/markets/polymarket/data_models.py +++ b/prediction_market_agent_tooling/markets/polymarket/data_models.py @@ -1,5 +1,3 @@ -from datetime import datetime - from pydantic import BaseModel from prediction_market_agent_tooling.gtypes import USDC, Probability, usdc_type @@ -10,13 +8,14 @@ PolymarketFullMarket, construct_polymarket_url, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class PolymarketRewards(BaseModel): min_size: int max_spread: float | None - event_start_date: datetime | None = None - event_end_date: datetime | None = None + event_start_date: DatetimeUTC | None = None + event_end_date: DatetimeUTC | None = None in_game_multiplier: int | None = None reward_epoch: int | None = None @@ -39,8 +38,8 @@ class PolymarketMarket(BaseModel): question: str description: str market_slug: str - end_date_iso: datetime | None - game_start_time: datetime | None + end_date_iso: DatetimeUTC | None + game_start_time: DatetimeUTC | None seconds_delay: int fpmm: str maker_base_fee: int diff --git a/prediction_market_agent_tooling/markets/polymarket/data_models_web.py b/prediction_market_agent_tooling/markets/polymarket/data_models_web.py index aa567267..698e8f07 100644 --- a/prediction_market_agent_tooling/markets/polymarket/data_models_web.py +++ b/prediction_market_agent_tooling/markets/polymarket/data_models_web.py @@ -7,7 +7,6 @@ import json import typing as t -from datetime import datetime import requests from pydantic import BaseModel, field_validator @@ -15,6 +14,7 @@ from prediction_market_agent_tooling.gtypes import USDC, HexAddress from prediction_market_agent_tooling.loggers import logger from prediction_market_agent_tooling.markets.data_models import Resolution +from prediction_market_agent_tooling.tools.utils import DatetimeUTC POLYMARKET_BASE_URL = "https://polymarket.com" POLYMARKET_TRUE_OUTCOME = "Yes" @@ -38,7 +38,7 @@ class Event(BaseModel): class Event1(BaseModel): - startDate: datetime | None = None + startDate: DatetimeUTC | None = None slug: str @@ -60,7 +60,7 @@ class Market1(BaseModel): class ResolutionData(BaseModel): id: str author: str - lastUpdateTimestamp: datetime + lastUpdateTimestamp: int status: str wasDisputed: bool price: str @@ -79,13 +79,13 @@ class Market(BaseModel): slug: str twitterCardImage: t.Any | None = None resolutionSource: str | None = None - endDate: datetime + endDate: DatetimeUTC category: t.Any | None = None ammType: t.Any | None = None description: str liquidity: str | None = None - startDate: datetime | None = None - createdAt: datetime + startDate: DatetimeUTC | None = None + createdAt: DatetimeUTC xAxisValue: t.Any | None = None yAxisValue: t.Any | None = None denominationToken: t.Any | None = None @@ -106,7 +106,7 @@ class Market(BaseModel): upperBoundDate: t.Any | None = None closed: bool marketMakerAddress: HexAddress - closedTime: datetime | None = None + closedTime: DatetimeUTC | None = None wideFormat: bool | None = None new: bool | None = None sentDiscord: t.Any | None = None @@ -129,9 +129,9 @@ class Market(BaseModel): curationOrder: t.Any | None = None volumeNum: USDC | None = None liquidityNum: float | None = None - endDateIso: datetime | None = None - startDateIso: datetime | None = None - umaEndDateIso: datetime | None = None + endDateIso: DatetimeUTC | None = None + startDateIso: DatetimeUTC | None = None + umaEndDateIso: DatetimeUTC | None = None commentsEnabled: bool | None = None disqusThread: t.Any | None = None gameStartTime: t.Any | None = None @@ -236,8 +236,8 @@ class PolymarketFullMarket(BaseModel): description: str commentCount: int | None = None resolutionSource: str | None = None - startDate: datetime | None = None - endDate: datetime + startDate: DatetimeUTC | None = None + endDate: DatetimeUTC image: str | None = None icon: str | None = None featuredImage: str | None = None @@ -253,10 +253,10 @@ class PolymarketFullMarket(BaseModel): competitive: float | None = None openInterest: int | None = None sortBy: str | None = None - createdAt: datetime + createdAt: DatetimeUTC commentsEnabled: bool | None = None disqusThread: t.Any | None = None - updatedAt: datetime + updatedAt: DatetimeUTC enableOrderBook: bool | None = None liquidityAmm: float | None = None liquidityClob: float | None = None diff --git a/prediction_market_agent_tooling/markets/polymarket/polymarket.py b/prediction_market_agent_tooling/markets/polymarket/polymarket.py index 60f5829f..8e0b5bcb 100644 --- a/prediction_market_agent_tooling/markets/polymarket/polymarket.py +++ b/prediction_market_agent_tooling/markets/polymarket/polymarket.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime from prediction_market_agent_tooling.markets.agent_market import ( AgentMarket, @@ -16,6 +15,7 @@ from prediction_market_agent_tooling.markets.polymarket.data_models_web import ( POLYMARKET_BASE_URL, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class PolymarketAgentMarket(AgentMarket): @@ -54,7 +54,7 @@ def get_binary_markets( limit: int, sort_by: SortBy = SortBy.NONE, filter_by: FilterBy = FilterBy.OPEN, - created_after: t.Optional[datetime] = None, + created_after: t.Optional[DatetimeUTC] = None, excluded_questions: set[str] | None = None, ) -> t.Sequence["PolymarketAgentMarket"]: if sort_by != SortBy.NONE: diff --git a/prediction_market_agent_tooling/monitor/markets/manifold.py b/prediction_market_agent_tooling/monitor/markets/manifold.py index 17148361..015e36b4 100644 --- a/prediction_market_agent_tooling/monitor/markets/manifold.py +++ b/prediction_market_agent_tooling/monitor/markets/manifold.py @@ -15,7 +15,7 @@ DeployedAgent, KubernetesCronJob, ) -from prediction_market_agent_tooling.tools.utils import DatetimeWithTimezone +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class DeployedManifoldAgent(DeployedAgent): @@ -57,7 +57,7 @@ def from_env_vars_without_prefix( @staticmethod def from_api_keys( name: str, - start_time: DatetimeWithTimezone, + start_time: DatetimeUTC, api_keys: APIKeys, ) -> "DeployedManifoldAgent": return DeployedManifoldAgent( diff --git a/prediction_market_agent_tooling/monitor/markets/metaculus.py b/prediction_market_agent_tooling/monitor/markets/metaculus.py index 5782d0b9..4e27e179 100644 --- a/prediction_market_agent_tooling/monitor/markets/metaculus.py +++ b/prediction_market_agent_tooling/monitor/markets/metaculus.py @@ -4,7 +4,7 @@ from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.constants import MARKET_TYPE_KEY -from prediction_market_agent_tooling.gtypes import DatetimeWithTimezone +from prediction_market_agent_tooling.gtypes import DatetimeUTC from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.markets import MarketType from prediction_market_agent_tooling.monitor.monitor import DeployedAgent @@ -23,7 +23,7 @@ def get_resolved_bets(self) -> list[ResolvedBet]: @staticmethod def from_api_keys( name: str, - start_time: DatetimeWithTimezone, + start_time: DatetimeUTC, api_keys: APIKeys, ) -> "DeployedMetaculusAgent": return DeployedMetaculusAgent( diff --git a/prediction_market_agent_tooling/monitor/markets/omen.py b/prediction_market_agent_tooling/monitor/markets/omen.py index a97c6dc1..72b082ef 100644 --- a/prediction_market_agent_tooling/monitor/markets/omen.py +++ b/prediction_market_agent_tooling/monitor/markets/omen.py @@ -4,7 +4,7 @@ from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.constants import MARKET_TYPE_KEY -from prediction_market_agent_tooling.gtypes import ChecksumAddress, DatetimeWithTimezone +from prediction_market_agent_tooling.gtypes import ChecksumAddress, DatetimeUTC from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.markets import MarketType from prediction_market_agent_tooling.markets.omen.omen_subgraph_handler import ( @@ -57,7 +57,7 @@ def from_env_vars_without_prefix( @staticmethod def from_api_keys( name: str, - start_time: DatetimeWithTimezone, + start_time: DatetimeUTC, api_keys: APIKeys, ) -> "DeployedOmenAgent": return DeployedOmenAgent( diff --git a/prediction_market_agent_tooling/monitor/markets/polymarket.py b/prediction_market_agent_tooling/monitor/markets/polymarket.py index 8cf1d05a..1b2502ff 100644 --- a/prediction_market_agent_tooling/monitor/markets/polymarket.py +++ b/prediction_market_agent_tooling/monitor/markets/polymarket.py @@ -4,7 +4,7 @@ from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.constants import MARKET_TYPE_KEY -from prediction_market_agent_tooling.gtypes import ChecksumAddress, DatetimeWithTimezone +from prediction_market_agent_tooling.gtypes import ChecksumAddress, DatetimeUTC from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.markets import MarketType from prediction_market_agent_tooling.monitor.monitor import DeployedAgent @@ -25,7 +25,7 @@ def get_resolved_bets(self) -> list[ResolvedBet]: @staticmethod def from_api_keys( name: str, - start_time: DatetimeWithTimezone, + start_time: DatetimeUTC, api_keys: APIKeys, ) -> "DeployedPolymarketAgent": return DeployedPolymarketAgent( diff --git a/prediction_market_agent_tooling/monitor/monitor.py b/prediction_market_agent_tooling/monitor/monitor.py index 6b3003ab..e1f4467f 100644 --- a/prediction_market_agent_tooling/monitor/monitor.py +++ b/prediction_market_agent_tooling/monitor/monitor.py @@ -8,7 +8,7 @@ import pandas as pd import streamlit as st from google.cloud.functions_v2.types.functions import Function -from pydantic import BaseModel, field_validator +from pydantic import BaseModel from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.gcp.kubernetes_models import ( @@ -26,9 +26,8 @@ from prediction_market_agent_tooling.markets.data_models import Resolution, ResolvedBet from prediction_market_agent_tooling.tools.parallelism import par_map from prediction_market_agent_tooling.tools.utils import ( - DatetimeWithTimezone, + DatetimeUTC, check_not_none, - convert_to_utc_datetime, should_not_happen, ) @@ -40,21 +39,12 @@ class DeployedAgent(BaseModel): name: str - start_time: DatetimeWithTimezone - end_time: t.Optional[ - DatetimeWithTimezone - ] = None # TODO: If we want end time, we need to store agents somewhere, not just query them from functions. + start_time: DatetimeUTC + end_time: DatetimeUTC | None = None # TODO: If we want end time, we need to store agents somewhere, not just query them from functions. raw_labels: dict[str, str] | None = None raw_env_vars: dict[str, str] | None = None - _add_timezone_validator_start_time = field_validator("start_time")( - convert_to_utc_datetime - ) - _add_timezone_validator_end_time = field_validator("end_time")( - convert_to_utc_datetime - ) - def model_dump_prefixed(self) -> dict[str, t.Any]: return { self.PREFIX + k: v for k, v in self.model_dump().items() if v is not None @@ -93,7 +83,7 @@ def from_env_vars( @staticmethod def from_api_keys( name: str, - start_time: DatetimeWithTimezone, + start_time: DatetimeUTC, api_keys: APIKeys, ) -> "DeployedAgent": raise NotImplementedError("Subclasses must implement this method.") diff --git a/prediction_market_agent_tooling/monitor/monitor_app.py b/prediction_market_agent_tooling/monitor/monitor_app.py index 7367746a..ad006789 100644 --- a/prediction_market_agent_tooling/monitor/monitor_app.py +++ b/prediction_market_agent_tooling/monitor/monitor_app.py @@ -1,7 +1,6 @@ import typing as t from datetime import date, datetime, timedelta -import pytz import streamlit as st from prediction_market_agent_tooling.markets.agent_market import ( @@ -27,9 +26,9 @@ ) from prediction_market_agent_tooling.monitor.monitor_settings import MonitorSettings from prediction_market_agent_tooling.tools.utils import ( - DatetimeWithTimezone, + DatetimeUTC, check_not_none, - convert_to_utc_datetime, + utc_datetime, utcnow, ) @@ -46,7 +45,7 @@ def get_deployed_agents( market_type: MarketType, settings: MonitorSettings, - start_time: DatetimeWithTimezone | None, + start_time: DatetimeUTC | None, ) -> list[DeployedAgent]: cls = MARKET_TYPE_TO_DEPLOYED_AGENT.get(market_type) if cls is None: @@ -76,7 +75,7 @@ def get_deployed_agents( def get_open_and_resolved_markets( - start_time: datetime, + start_time: DatetimeUTC, market_type: MarketType, ) -> tuple[t.Sequence[AgentMarket], t.Sequence[AgentMarket]]: cls = market_type.market_class @@ -103,8 +102,8 @@ def monitor_app( market_type: MarketType = check_not_none( st.selectbox(label="Market type", options=enabled_market_types, index=0) ) - start_time: DatetimeWithTimezone | None = ( - convert_to_utc_datetime( + start_time: DatetimeUTC | None = ( + DatetimeUTC.from_datetime( datetime.combine( t.cast( # This will be always a date for us, so casting. @@ -131,7 +130,7 @@ def monitor_app( oldest_start_time = ( min(agent.start_time for agent in agents) if agents - else datetime(2020, 1, 1, tzinfo=pytz.UTC) + else utc_datetime(2020, 1, 1) ) st.header("Market Info") diff --git a/prediction_market_agent_tooling/tools/contract.py b/prediction_market_agent_tooling/tools/contract.py index c97441dd..609bf8ba 100644 --- a/prediction_market_agent_tooling/tools/contract.py +++ b/prediction_market_agent_tooling/tools/contract.py @@ -22,11 +22,7 @@ GNOSIS_NETWORK_ID, GNOSIS_RPC_URL, ) -from prediction_market_agent_tooling.tools.utils import ( - DatetimeWithTimezone, - should_not_happen, - utc_timestamp_to_utc_datetime, -) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, should_not_happen from prediction_market_agent_tooling.tools.web3_utils import ( call_function_on_contract, send_function_on_contract_tx, @@ -457,8 +453,8 @@ def getNow( def get_now( self, web3: Web3 | None = None, - ) -> DatetimeWithTimezone: - return utc_timestamp_to_utc_datetime(self.getNow(web3)) + ) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.getNow(web3)) def inc( self, diff --git a/prediction_market_agent_tooling/tools/datetime_utc.py b/prediction_market_agent_tooling/tools/datetime_utc.py new file mode 100644 index 00000000..4308f13d --- /dev/null +++ b/prediction_market_agent_tooling/tools/datetime_utc.py @@ -0,0 +1,74 @@ +import typing as t +from datetime import datetime, timedelta + +import pytz +from dateutil import parser +from pydantic import GetCoreSchemaHandler +from pydantic_core import CoreSchema, core_schema + +from prediction_market_agent_tooling.loggers import logger + + +class DatetimeUTC(datetime): + """ + As a subclass of `datetime` instead of `NewType` because otherwise it doesn't work with issubclass command which is required for SQLModel/Pydantic. + """ + + def __new__(cls, *args, **kwargs) -> "DatetimeUTC": # type: ignore[no-untyped-def] # Pickling doesn't work if I copy-paste arguments from datetime's __new__. + if len(args) >= 8: + # Start of Selection + args = args[:7] + (pytz.UTC,) + args[8:] + else: + kwargs["tzinfo"] = pytz.UTC + return super().__new__(cls, *args, **kwargs) + + @classmethod + def _validate(cls, value: t.Any) -> "DatetimeUTC": + if not isinstance(value, (datetime, int, str)): + raise TypeError( + f"Expected datetime, timestamp or string, got {type(value)}" + ) + return cls.to_datetime_utc(value) + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: t.Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + dt_schema = handler(datetime) + return core_schema.no_info_after_validator_function(cls._validate, dt_schema) + + @staticmethod + def from_datetime(dt: datetime) -> "DatetimeUTC": + """ + Converts a datetime object to DatetimeUTC, ensuring it is timezone-aware in UTC. + """ + if dt.tzinfo is None: + logger.warning( + f"tzinfo not provided, assuming the timezone of {dt=} is UTC." + ) + dt = dt.replace(tzinfo=pytz.UTC) + else: + dt = dt.astimezone(pytz.UTC) + return DatetimeUTC( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + tzinfo=pytz.UTC, + ) + + @staticmethod + def to_datetime_utc(value: datetime | int | str) -> "DatetimeUTC": + if isinstance(value, int): + # Divide by 1000 if the timestamp is assumed to be in miliseconds (if not, 1e11 would be year 5138). + value = int(value / 1000) if value > 1e11 else value + # In the past, we had bugged data where timestamp was huge and Python errored out. + max_timestamp = int((datetime.max - timedelta(days=1)).timestamp()) + value = min(value, max_timestamp) + value = datetime.fromtimestamp(value, tz=pytz.UTC) + elif isinstance(value, str): + value = parser.parse(value) + return DatetimeUTC.from_datetime(value) diff --git a/prediction_market_agent_tooling/tools/langfuse_client_utils.py b/prediction_market_agent_tooling/tools/langfuse_client_utils.py index dba37930..4452662b 100644 --- a/prediction_market_agent_tooling/tools/langfuse_client_utils.py +++ b/prediction_market_agent_tooling/tools/langfuse_client_utils.py @@ -1,5 +1,4 @@ import typing as t -from datetime import datetime import numpy as np from langfuse import Langfuse @@ -14,15 +13,19 @@ TradeType, ) from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket -from prediction_market_agent_tooling.tools.utils import convert_to_utc_datetime +from prediction_market_agent_tooling.tools.utils import DatetimeUTC class ProcessMarketTrace(BaseModel): - timestamp: datetime + timestamp: int market: OmenAgentMarket answer: ProbabilisticAnswer trades: list[PlacedTrade] + @property + def timestamp_datetime(self) -> DatetimeUTC: + return DatetimeUTC.to_datetime_utc(self.timestamp) + @property def buy_trade(self) -> PlacedTrade | None: buy_trades = [t for t in self.trades if t.trade_type == TradeType.BUY] @@ -45,7 +48,7 @@ def from_langfuse_trace( market=market, answer=answer, trades=trades, - timestamp=trace.timestamp, + timestamp=int(trace.timestamp.timestamp()), ) @@ -57,7 +60,7 @@ class ResolvedBetWithTrace(BaseModel): def get_traces_for_agent( agent_name: str, trace_name: str, - from_timestamp: datetime, + from_timestamp: DatetimeUTC, has_output: bool, client: Langfuse, ) -> list[TraceWithDetails]: @@ -115,7 +118,7 @@ def trace_to_trades(trace: TraceWithDetails) -> list[PlacedTrade]: def get_closest_datetime_from_list( - ref_datetime: datetime, datetimes: list[datetime] + ref_datetime: DatetimeUTC, datetimes: list[DatetimeUTC] ) -> int: """Get the index of the closest datetime to the reference datetime""" if len(datetimes) == 1: @@ -149,20 +152,22 @@ def get_trace_for_bet( else: # In-case there are multiple traces for the same market, get the closest # trace to the bet - bet_timestamp = convert_to_utc_datetime(bet.created_time) closest_trace_index = get_closest_datetime_from_list( - bet_timestamp, - [t.timestamp for t in traces_for_bet], + bet.created_time, + [t.timestamp_datetime for t in traces_for_bet], ) # Sanity check: Let's say the upper bound for time between # `agent.process_market` being called and the bet being placed is 20 # minutes candidate_trace = traces_for_bet[closest_trace_index] - if abs(candidate_trace.timestamp - bet_timestamp).total_seconds() > 1200: + if ( + abs(candidate_trace.timestamp_datetime - bet.created_time).total_seconds() + > 1200 + ): logger.info( f"Closest trace to bet has timestamp {candidate_trace.timestamp}, " - f"but bet was created at {bet_timestamp}. Not matching" + f"but bet was created at {bet.created_time}. Not matching" ) return None diff --git a/prediction_market_agent_tooling/tools/tavily_storage/tavily_models.py b/prediction_market_agent_tooling/tools/tavily_storage/tavily_models.py index 491082c7..4eda80b2 100644 --- a/prediction_market_agent_tooling/tools/tavily_storage/tavily_models.py +++ b/prediction_market_agent_tooling/tools/tavily_storage/tavily_models.py @@ -1,5 +1,5 @@ import typing as t -from datetime import datetime, timedelta +from datetime import timedelta import tenacity from pydantic import BaseModel @@ -18,7 +18,7 @@ from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.loggers import logger -from prediction_market_agent_tooling.tools.utils import utcnow +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, utcnow class TavilyResult(BaseModel): @@ -59,7 +59,7 @@ class TavilyResponseModel(SQLModel, table=True): include_images: bool use_cache: bool # Datetime at the time of search response and response from the search - datetime_: datetime = Field(index=True, nullable=False) + datetime_: DatetimeUTC = Field(index=True, nullable=False) response: dict[str, t.Any] = Field(sa_column=Column(JSONB, nullable=False)) @staticmethod @@ -89,7 +89,7 @@ def from_model( include_raw_content=include_raw_content, include_images=include_images, use_cache=use_cache, - datetime_=datetime.now(), + datetime_=utcnow(), response=response.model_dump(), ) diff --git a/prediction_market_agent_tooling/tools/utils.py b/prediction_market_agent_tooling/tools/utils.py index 440dbbb9..d0c9876e 100644 --- a/prediction_market_agent_tooling/tools/utils.py +++ b/prediction_market_agent_tooling/tools/utils.py @@ -1,20 +1,18 @@ import json import os import subprocess -import typing as t from datetime import datetime -from typing import Any, NoReturn, Optional, Type, TypeVar, cast +from typing import Any, NoReturn, Optional, Type, TypeVar import pytz import requests from google.cloud import secretmanager from pydantic import BaseModel, ValidationError -from pydantic.functional_validators import BeforeValidator from scipy.optimize import newton from scipy.stats import entropy from prediction_market_agent_tooling.gtypes import ( - DatetimeWithTimezone, + DatetimeUTC, PrivateKey, Probability, SecretStr, @@ -85,55 +83,33 @@ def export_requirements_from_toml(output_dir: str) -> None: logger.debug(f"Saved requirements to {output_dir}/requirements.txt") -@t.overload -def convert_to_utc_datetime(value: datetime) -> DatetimeWithTimezone: - ... - - -@t.overload -def convert_to_utc_datetime(value: None) -> None: - ... - - -def convert_to_utc_datetime(value: datetime | None) -> DatetimeWithTimezone | None: - """ - If datetime doesn't come with a timezone, we assume it to be UTC. - Note: Not great, but at least the error will be constant. - """ - if value is None: - return None - if value.tzinfo is None: - value = value.replace(tzinfo=pytz.UTC) - if value.tzinfo != pytz.UTC: - value = value.astimezone(pytz.UTC) - return cast(DatetimeWithTimezone, value) - - -@t.overload -def utc_timestamp_to_utc_datetime(ts: int) -> DatetimeWithTimezone: - ... - - -@t.overload -def utc_timestamp_to_utc_datetime(ts: None) -> None: - ... - - -def utc_timestamp_to_utc_datetime(ts: int | None) -> DatetimeWithTimezone | None: - return ( - convert_to_utc_datetime(datetime.fromtimestamp(ts, tz=pytz.UTC)) - if ts is not None - else None +def utcnow() -> DatetimeUTC: + return DatetimeUTC.from_datetime(datetime.now(pytz.UTC)) + + +def utc_datetime( + year: int, + month: int, + day: int, + hour: int = 0, + minute: int = 0, + second: int = 0, + microsecond: int = 0, + *, + fold: int = 0, +) -> DatetimeUTC: + dt = datetime( + year=year, + month=month, + day=day, + hour=hour, + minute=minute, + second=second, + microsecond=microsecond, + tzinfo=pytz.UTC, + fold=fold, ) - - -UTCDatetimeFromUTCTimestamp = t.Annotated[ - datetime, BeforeValidator(utc_timestamp_to_utc_datetime) -] - - -def utcnow() -> DatetimeWithTimezone: - return convert_to_utc_datetime(datetime.now(pytz.UTC)) + return DatetimeUTC.from_datetime(dt) def get_current_git_commit_sha() -> str: diff --git a/pyproject.toml b/pyproject.toml index 40c58bda..07dc6e11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prediction-market-agent-tooling" -version = "0.49.2" +version = "0.50.0" description = "Tools to benchmark, deploy and monitor prediction market agents." authors = ["Gnosis"] readme = "README.md" @@ -44,10 +44,12 @@ langfuse = "^2.42.0" openai = { version = "^1.0.0", optional = true} pymongo = "^4.8.0" tavily-python = "^0.3.9" -sqlmodel = "^0.0.21" +sqlmodel = "^0.0.22" psycopg2-binary = "^2.9.9" base58 = ">=1.0.2,<2.0" loky = "^3.4.1" +python-dateutil = "^2.9.0.post0" +types-python-dateutil = "^2.9.0.20240906" pinatapy-vourhey = "^0.2.0" hishel = "^0.0.31" diff --git a/scripts/create_market_omen.py b/scripts/create_market_omen.py index 24fd237f..67771645 100644 --- a/scripts/create_market_omen.py +++ b/scripts/create_market_omen.py @@ -1,5 +1,3 @@ -from datetime import datetime - import typer from web3 import Web3 @@ -15,11 +13,12 @@ OMEN_DEFAULT_MARKET_FEE_PERC, CollateralTokenChoice, ) +from prediction_market_agent_tooling.tools.utils import DatetimeUTC def main( question: str = typer.Option(), - closing_time: datetime = typer.Option(), + closing_time: DatetimeUTC = typer.Option(), category: str = typer.Option(), initial_funds: str = typer.Option(), from_private_key: str = typer.Option(), diff --git a/tests/markets/omen/test_omen.py b/tests/markets/omen/test_omen.py index 2cd155b3..e61904b9 100644 --- a/tests/markets/omen/test_omen.py +++ b/tests/markets/omen/test_omen.py @@ -1,17 +1,10 @@ -from datetime import datetime - import numpy as np import pytest from eth_account import Account from eth_typing import HexAddress, HexStr from web3 import Web3 -from prediction_market_agent_tooling.gtypes import ( - DatetimeWithTimezone, - OutcomeStr, - Wei, - xDai, -) +from prediction_market_agent_tooling.gtypes import OutcomeStr, Wei, xDai from prediction_market_agent_tooling.markets.agent_market import FilterBy, SortBy from prediction_market_agent_tooling.markets.data_models import Position, TokenAmount from prediction_market_agent_tooling.markets.omen.data_models import ( @@ -30,7 +23,11 @@ from prediction_market_agent_tooling.tools.betting_strategies.market_moving import ( get_market_moving_bet, ) -from prediction_market_agent_tooling.tools.utils import check_not_none, utcnow +from prediction_market_agent_tooling.tools.utils import ( + check_not_none, + utc_datetime, + utcnow, +) from prediction_market_agent_tooling.tools.web3_utils import wei_to_xdai @@ -81,9 +78,7 @@ def test_omen_market_close_time() -> None: assert ( market.close_time >= time_now ), "Market close time should be in the future." - time_now = DatetimeWithTimezone( - market.close_time - ) # Ensure close time is in ascending order + time_now = market.close_time # Ensure close time is in ascending order def test_market_liquidity() -> None: @@ -193,8 +188,8 @@ def test_positions_value() -> None: "0x2DD9f5678484C1F59F97eD334725858b938B4102" ) resolved_bets = OmenSubgraphHandler().get_resolved_bets_with_valid_answer( - start_time=datetime(2024, 3, 27, 4, 20), - end_time=datetime(2024, 3, 27, 4, 30), + start_time=utc_datetime(2024, 3, 27, 4, 20), + end_time=utc_datetime(2024, 3, 27, 4, 30), better_address=user_address, ) assert len(resolved_bets) == 1 diff --git a/tests/markets/omen/test_omen_subgraph_handler.py b/tests/markets/omen/test_omen_subgraph_handler.py index 9ba95cc5..eae5f9fa 100644 --- a/tests/markets/omen/test_omen_subgraph_handler.py +++ b/tests/markets/omen/test_omen_subgraph_handler.py @@ -1,5 +1,4 @@ import sys -from datetime import datetime import pytest from eth_typing import HexAddress, HexStr @@ -15,6 +14,7 @@ OmenSubgraphHandler, ) from prediction_market_agent_tooling.tools.hexbytes_custom import HexBytes +from prediction_market_agent_tooling.tools.utils import DatetimeUTC, utc_datetime MARKET_ID_WITH_SDAI_AS_COLLATERAL = "0x4ecb20cea4d1b0c90d935a45213d27e1695bee92" @@ -45,8 +45,8 @@ def test_markets_with_creation_timestamp_between( bets = omen_subgraph_handler.get_bets( better_address=Web3.to_checksum_address(creator), filter_by_answer_finalized_not_null=False, - start_time=datetime.fromtimestamp(1625073159), - end_time=datetime.fromtimestamp(1625073162), + start_time=DatetimeUTC.to_datetime_utc(1625073159), + end_time=DatetimeUTC.to_datetime_utc(1625073162), ) assert len(bets) == 1 bet = bets[0] @@ -76,8 +76,8 @@ def test_resolved_omen_bets( ) -> None: better_address = Web3.to_checksum_address(a_bet_from_address) resolved_bets = omen_subgraph_handler.get_resolved_bets_with_valid_answer( - start_time=datetime(2024, 2, 20), - end_time=datetime(2024, 2, 28), + start_time=utc_datetime(2024, 2, 20), + end_time=utc_datetime(2024, 2, 28), better_address=better_address, ) @@ -95,8 +95,8 @@ def test_get_bets( ) -> None: better_address = Web3.to_checksum_address(a_bet_from_address) bets = omen_subgraph_handler.get_bets( - start_time=datetime(2024, 2, 20), - end_time=datetime(2024, 2, 21), + start_time=utc_datetime(2024, 2, 20), + end_time=utc_datetime(2024, 2, 21), better_address=better_address, ) assert len(bets) == 1 diff --git a/tests/markets/test_manifold.py b/tests/markets/test_manifold.py index dc4b01fc..e46d8543 100644 --- a/tests/markets/test_manifold.py +++ b/tests/markets/test_manifold.py @@ -1,7 +1,6 @@ -from datetime import datetime, timedelta +from datetime import timedelta import pytest -import pytz from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.gtypes import mana_type @@ -16,6 +15,7 @@ place_bet, ) from prediction_market_agent_tooling.markets.manifold.data_models import ManifoldPool +from prediction_market_agent_tooling.tools.utils import utc_datetime from tests.utils import RUN_PAID_TESTS @@ -47,7 +47,7 @@ def test_manifold_full_market() -> None: def test_manifold_bets(a_user_id: str) -> None: - start_time = datetime(2020, 2, 1, tzinfo=pytz.UTC) + start_time = utc_datetime(2020, 2, 1) bets = get_manifold_bets( user_id=a_user_id, start_time=start_time, @@ -57,7 +57,7 @@ def test_manifold_bets(a_user_id: str) -> None: def test_resolved_manifold_bets(a_user_id: str) -> None: - start_time = datetime(2024, 2, 20, tzinfo=pytz.UTC) + start_time = utc_datetime(2024, 2, 20) resolved_bets, markets = get_resolved_manifold_bets( user_id=a_user_id, start_time=start_time, diff --git a/tests/markets/test_markets.py b/tests/markets/test_markets.py index 7f506a23..8c711b20 100644 --- a/tests/markets/test_markets.py +++ b/tests/markets/test_markets.py @@ -74,3 +74,12 @@ def test_get_pool_tokens(market_type: MarketType) -> None: for outcome in market.outcomes: # Sanity check assert market.get_pool_tokens(outcome) > 0 + + +@pytest.mark.parametrize("market_type", list(MarketType)) +def test_get_markets(market_type: MarketType) -> None: + limit = 10 + markets = market_type.market_class.get_binary_markets( + limit=limit, sort_by=SortBy.NONE, filter_by=FilterBy.OPEN + ) + assert len(markets) <= limit diff --git a/tests/tools/test_datetime_utc.py b/tests/tools/test_datetime_utc.py new file mode 100644 index 00000000..0e81b7bb --- /dev/null +++ b/tests/tools/test_datetime_utc.py @@ -0,0 +1,52 @@ +import pickle +from datetime import datetime, timedelta + +import pytest +import pytz + +from prediction_market_agent_tooling.tools.datetime_utc import DatetimeUTC +from prediction_market_agent_tooling.tools.utils import utcnow + + +@pytest.mark.parametrize( + "value, expected", + [ + ("2024-10-10", DatetimeUTC(2024, 10, 10)), + ("2024-10-10T17:00+02:00", DatetimeUTC(2024, 10, 10, 15)), + ("2003 Sep 25", DatetimeUTC(2003, 9, 25)), + ( + 1727879129, + DatetimeUTC( + 2024, + 10, + 2, + 14, + 25, + 29, + ), + ), + ], +) +def test_datetime_utc(value: str | int | datetime, expected: DatetimeUTC) -> None: + assert DatetimeUTC.to_datetime_utc(value) == expected + + +def test_datetime_utc_pickle() -> None: + now = utcnow() + dumped = pickle.dumps(now) + loaded = pickle.loads(dumped) + assert now == loaded + + +def test_datetime_utc_is_utc() -> None: + now = utcnow() + assert isinstance(now, DatetimeUTC) + assert now.tzinfo == pytz.UTC + + +def test_datetime_utc_with_timedelta() -> None: + now = utcnow() + then = now + timedelta(hours=12) + assert then > now + assert type(now) == type(then) + assert isinstance(then, DatetimeUTC)