Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

. #77

Closed
wants to merge 1 commit into from
Closed

. #77

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions app/Network.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import logger, { updateSentryScope } from './services/Logger.js';
import ContractEventsEmitter from './services/ContractEventsEmitter.js';
import { SubgraphSource } from './dataSources/SubgraphSource.js';
import { SubquerySource } from './dataSources/SubquerySource.js';
import { BlockchainSource } from './dataSources/BlockchainSource.js';
import { AgentRandao_2_3_0 } from './agents/Agent.2.3.0.randao.js';
import { AgentLight_2_2_0 } from './agents/Agent.2.2.0.light.js';
Expand Down Expand Up @@ -408,6 +409,8 @@
);
if (agent.dataSourceType === 'subgraph') {
dataSource = this.getAgentSubgraphDataSource(agent);
} else if (agent.dataSourceType === 'subquery') { //allows subquery source alongside the subgraph one

Check failure on line 412 in app/Network.ts

View workflow job for this annotation

GitHub Actions / build

Insert `⏎·······`
dataSource = this.getAgentSubqueryDataSource(agent);
} else if (agent.dataSourceType === 'blockchain') {
dataSource = this.getAgentBlockchainDataSource(agent);
} else {
Expand All @@ -423,6 +426,10 @@
return new SubgraphSource(this, agent, agent.subgraphUrl);
}

public getAgentSubqueryDataSource(agent) {
return new SubquerySource(this, agent, agent.subgraphUrl);
}

public getAgentBlockchainDataSource(agent) {
return new BlockchainSource(this, agent);
}
Expand Down
2 changes: 1 addition & 1 deletion app/Types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { AbstractJob } from './jobs/AbstractJob';
export type AvailableNetworkNames = 'mainnet' | 'bsc' | 'polygon' | 'goerli';
export type ExecutorType = 'flashbots' | 'pga';
export type Strategy = 'randao' | 'light';
export type DataSourceType = 'blockchain' | 'subgraph';
export type DataSourceType = 'blockchain' | 'subgraph' | 'subquery';

export enum CALLDATA_SOURCE {
SELECTOR,
Expand Down
8 changes: 8 additions & 0 deletions app/agents/AbstractAgent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,14 @@ export abstract class AbstractAgent implements IAgent {
}
this.dataSourceType = 'subgraph';
this.subgraphUrl = agentConfig.subgraph_url;
} else if (agentConfig.data_source === 'subquery') {
if (!agentConfig.subgraph_url) {
throw new Error(
"Please set 'subgraph_url' if you want to proceed with {'data_source': 'subquery'}. Notice that 'graph_url' is deprecated so please change it to 'subgraph_url'.",
);
}
this.dataSourceType = 'subquery';
this.subgraphUrl = agentConfig.subgraph_url;
} else if (agentConfig.data_source === 'blockchain') {
this.dataSourceType = 'blockchain';
} else {
Expand Down
2 changes: 1 addition & 1 deletion app/dataSources/SubgraphSource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,4 +253,4 @@
}
return { data: result, meta: graphStatus };
}
}
}

Check failure on line 256 in app/dataSources/SubgraphSource.ts

View workflow job for this annotation

GitHub Actions / build

Insert `⏎`
262 changes: 262 additions & 0 deletions app/dataSources/SubquerySource.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,262 @@
import axios from 'axios';
import { AbstractSource } from './AbstractSource.js';
import { BlockchainSource } from './BlockchainSource.js';
import { RandaoJob } from '../jobs/RandaoJob';
import { LightJob } from '../jobs/LightJob';
import { Network } from '../Network';
import { IAgent, SourceMetadata } from '../Types';
import { BigNumber, utils } from 'ethers';
import { toChecksummedAddress } from '../Utils.js';
import logger from '../services/Logger.js';
import { getMaxBlocksSubgraphDelay } from '../ConfigGetters.js';

//the different structure mirrors the structure of the subquery database, made this way due to a lack of priori documentation on subgraph
//everything else mirrors the subgraph exactly
export const QUERY_ALL_JOBS = `{
jobs(first: 1000) {
nodes {
id
active
jobAddress
jobId
assertResolverSelector
credits
calldataSource
fixedReward
jobSelector
lastExecutionAt
minKeeperCVP
preDefinedCalldata
intervalSeconds
resolverAddress
resolverCalldata
useJobOwnerCredits
owner {
id
}
pendingOwner {
id
}
jobCreatedAt
jobNextKeeperId
jobReservedSlasherId
jobSlashingPossibleAfter
}
}
}`;

export const QUERY_META = `{
_metadata {
lastProcessedHeight
}
}`;

export const QUERY_JOB_OWNERS = `{
jobOwners {
nodes {
id
credits
}
}
}`;

/**
* This class used for fetching data from subgraph
*/
export class SubquerySource extends AbstractSource {
private blockchainSource: BlockchainSource;
private readonly subgraphUrl: string;

private toString(): string {
return `(url: ${this.subgraphUrl})`;
}

private clog(level: string, ...args: unknown[]) {
logger.log(level, `SubgraphDataSource${this.toString()}: ${args.join(' ')}`);
}

private err(...args: unknown[]): Error {
return new Error(`SubgraphDataSourceError${this.toString()}: ${args.join(' ')}`);
}

constructor(network: Network, agent: IAgent, graphUrl: string) {
super(network, agent);
this.type = 'subgraph';
this.subgraphUrl = graphUrl;

this.blockchainSource = new BlockchainSource(network, agent);
}

/**
* A query builder for requests.
* It's using axios because "fetch" is bad at error handling (all error codes except network error is 200).
* @param endpoint
* @param query
*/
async query(endpoint, query) {
const res = await axios.post(endpoint, { query });
if (res.data.errors) {
let locations = '';
if ('locations' in res.data.errors[0]) {
locations = `Locations: ${JSON.stringify(res.data.errors[0].locations)}. `;
}
throw new Error(`Subgraph query error: ${res.data.errors[0].message}. ${locations}Executed query:\n${query}\n`);
}
return res.data.data;
}

/**
* Checking if our graph is existing and synced
*/
async getGraphStatus(): Promise<SourceMetadata> {
try {
const { diff, nodeBlockNumber, sourceBlockNumber } = await this.getBlocksDelay();
const isSynced = diff <= getMaxBlocksSubgraphDelay(this.network.getName()); // Our graph is desynced if its behind for more than 10 blocks
if (!isSynced) {
this.clog('error', `Subgraph is ${diff} blocks behind.`);
}
return { isSynced, diff, nodeBlockNumber, sourceBlockNumber };
} catch (e) {
this.clog('error', 'Graph meta query error:', e);
return { isSynced: false, diff: null, nodeBlockNumber: null, sourceBlockNumber: null };
}
}

async getBlocksDelay(): Promise<{ diff: bigint; nodeBlockNumber: bigint; sourceBlockNumber: bigint }> {
const [latestBock, { _metadata }] = await Promise.all([
this.network.getLatestBlockNumber(),
this.query(this.subgraphUrl, QUERY_META),
]);
return {
diff: latestBock - BigInt(_metadata.lastProcessedHeight),
nodeBlockNumber: latestBock,
sourceBlockNumber: latestBock,
};
}

/**
* Getting a list of jobs from subgraph and initialise job.
* Returns Map structure which key is jobKey and value is instance of RandaoJob or LightJob. Await is required.
*
* @param context - agent caller context. This can be Agent.2.2.0.light or Agent.2.3.0.randao
*
* @return Promise<Map<string, RandaoJob | LightJob>>
*/
async getRegisteredJobs(context): Promise<{ data: Map<string, RandaoJob | LightJob>; meta: SourceMetadata }> {
let newJobs = new Map<string, RandaoJob | LightJob>();
const graphStatus = await this.getGraphStatus();
if (!graphStatus.isSynced) {
this.clog('warn', 'Subgraph is not ok, falling back to the blockchain datasource.');
({ data: newJobs } = await this.blockchainSource.getRegisteredJobs(context));
return { data: newJobs, meta: graphStatus };
}
try {
const res = await this.query(this.subgraphUrl, QUERY_ALL_JOBS);
const { jobs } = res;
const { nodes } = jobs;
nodes.forEach(job => {
const newJob = context._buildNewJob({
name: 'RegisterJob',
args: {
jobAddress: job.jobAddress,
jobId: BigNumber.from(job.jobId),
jobKey: job.id,
},
});
const lensJob = this.addLensFieldsToJobs(job);
newJob.applyJob({
...lensJob,
owner: lensJob.owner,
config: lensJob.config,
});
newJobs.set(job.id, newJob);
});
} catch (e) {
throw this.err(e);
}
return { data: newJobs, meta: graphStatus };
}

/**
* here we can populate job with full graph data, as if we made a request to getJobs lens method.
* But we already hale all the data
* @param graphData
*/
private addLensFieldsToJobs(graphData) {
const lensFields: any = {};
// setting an owner
lensFields.owner = utils.getAddress(this._checkNullAddress(graphData.owner, true, 'id'));
// if job is about to get transferred setting future owner address. Otherwise, null address
lensFields.pendingTransfer = this._checkNullAddress(graphData.pendingOwner, true, 'id');
// transfer min cvp into bigNumber as it's returned in big number when getting data from blockchain. Data consistency.
lensFields.jobLevelMinKeeperCvp = BigNumber.from(graphData.minKeeperCVP);
// From graph zero predefinedcalldata is returned as null, but from blockchain its 0x
lensFields.preDefinedCalldata = this._checkNullAddress(graphData.preDefinedCalldata);

// setting a resolver field
lensFields.resolver = {
resolverCalldata: this._checkNullAddress(graphData.resolverCalldata),
resolverAddress: this._checkNullAddress(graphData.resolverAddress, true),
};
// setting randao data
lensFields.randaoData = {
jobNextKeeperId: BigNumber.from(graphData.jobNextKeeperId),
jobReservedSlasherId: BigNumber.from(graphData.jobReservedSlasherId),
jobSlashingPossibleAfter: BigNumber.from(graphData.jobSlashingPossibleAfter),
jobCreatedAt: BigNumber.from(graphData.jobCreatedAt),
};
// setting details
lensFields.details = {
selector: graphData.jobSelector,
credits: BigNumber.from(graphData.credits),
maxBaseFeeGwei: parseInt(graphData.maxBaseFeeGwei),
rewardPct: parseInt(graphData.rewardPct),
fixedReward: parseInt(graphData.fixedReward),
calldataSource: parseInt(graphData.calldataSource),
intervalSeconds: parseInt(graphData.intervalSeconds),
lastExecutionAt: parseInt(graphData.lastExecutionAt),
};
// with SubquerySource you don't need to use parseConfig. You can create config field right here.
lensFields.config = {
isActive: graphData.active,
useJobOwnerCredits: graphData.useJobOwnerCredits,
assertResolverSelector: graphData.assertResolverSelector,
checkKeeperMinCvpDeposit: +graphData.minKeeperCVP > 0,
};
return lensFields;
}

public async addLensFieldsToOneJob(newJob: LightJob | RandaoJob) {
return this.blockchainSource.addLensFieldsToOneJob(newJob);
}

/**
* Gets job owner's balances from subgraph
* @param context - agent context
* @param jobOwnersSet - array of jobOwners addresses
*/
public async getOwnersBalances(
context,
jobOwnersSet: Set<string>,
): Promise<{ data: Map<string, BigNumber>; meta: SourceMetadata }> {
let result = new Map<string, BigNumber>(),
graphStatus;
try {
graphStatus = await this.getGraphStatus();
if (!graphStatus.isSynced) {
this.clog('warn', 'Subgraph is not ok, falling back to the blockchain datasource.');
({ data: result } = await this.blockchainSource.getOwnersBalances(context, jobOwnersSet));
return { data: result, meta: graphStatus };
}

const { jobOwners } = await this.query(this.subgraphUrl, QUERY_JOB_OWNERS);
const { nodes } = jobOwners;
nodes.forEach(JobOwner => {
result.set(toChecksummedAddress(JobOwner.id), BigNumber.from(JobOwner.credits));
});
} catch (e) {
throw this.err(e);
}
return { data: result, meta: graphStatus };
}
}
Loading