Skip to content

Okex sequence info #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marcinvalas wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Okex sequence info #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 76 additions & 21 deletions src/mappers/okex.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { getOkexOptionsFamilyOrIndex, getOkexOptionsUnderlyingIndex } from '../o
import { BookChange, BookTicker, DerivativeTicker, Exchange, Liquidation, OptionSummary, Trade } from '../types.ts'
import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
import { OkexSpreadsBookChangeMapper, OkexSpreadsBookTickerMapper, OkexSpreadsTradesMapper } from './okexspreads.ts'
import { exchangeMappers, mapper } from './registry.ts'
import { exchangeMappers, isRealTime, mapper } from './registry.ts'

const OKEX_V5_API_SWITCH_DATE = new Date('2021-12-23T00:00:00.000Z')
const OKEX_V5_TBT_BOOK_TICKER_RELEASE_DATE = new Date('2022-05-06T00:00:00.000Z')
Expand Down Expand Up @@ -77,7 +77,9 @@ export const okexMappers = exchangeMappers({
until: OKCOIN_V5_API_SWITCH_DATE,
use: () => new OkexBookChangeMapper('okcoin', { market: 'spot', useTickByTickChannel: true })
},
{ use: () => new OkexV5BookChangeMapper('okcoin', { allowPublicBooksChannel: true }) }
{
use: () => new OkexV5BookChangeMapper('okcoin', { allowPublicBooksChannel: true, validateSequence: false })
}
]),
bookTickers: mapper([
{ until: OKCOIN_V5_API_SWITCH_DATE, use: () => new OkexBookTickerMapper('okcoin', { market: 'spot' }) },
Expand Down Expand Up @@ -106,10 +108,25 @@ function okexBookChangesMapper(
return mapper([
{ until: depthL2TbtSwitchDate, use: () => new OkexBookChangeMapper(exchange, { market, useTickByTickChannel: false }) },
{ until: OKEX_V5_API_SWITCH_DATE, use: () => new OkexBookChangeMapper(exchange, { market, useTickByTickChannel: true }) },
{ until: OKX_PUBLIC_BOOKS_CHANNEL_START_DATE, use: () => new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: false }) },
{ until: OKX_PUBLIC_BOOKS_CHANNEL_END_DATE, use: () => new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: true }) },
{ until: OKEX_PUBLIC_BOOKS_SWITCH_DATE, use: () => new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: false }) },
{ use: () => new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: true }) }
{
until: OKX_PUBLIC_BOOKS_CHANNEL_START_DATE,
use: (localTimestamp) =>
new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: false, validateSequence: isRealTime(localTimestamp) })
},
{
until: OKX_PUBLIC_BOOKS_CHANNEL_END_DATE,
use: (localTimestamp) =>
new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: true, validateSequence: isRealTime(localTimestamp) })
},
{
until: OKEX_PUBLIC_BOOKS_SWITCH_DATE,
use: (localTimestamp) =>
new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: false, validateSequence: isRealTime(localTimestamp) })
},
{
use: (localTimestamp) =>
new OkexV5BookChangeMapper(exchange, { allowPublicBooksChannel: true, validateSequence: isRealTime(localTimestamp) })
}
])
}

Expand Down Expand Up @@ -187,12 +204,18 @@ const mapV5BookLevel = (level: OkexV5BookLevel) => {

class OkexV5BookChangeMapper implements Mapper<OKEX_EXCHANGES, BookChange> {
private _channelName: string
private readonly lastSequenceIdBySymbol = new Map<string, number>()
private readonly validateSequenceContinuity: boolean
private readonly _hasCredentials = process.env.OKX_API_KEY !== undefined
private readonly _hasVip5Access = process.env.OKX_API_VIP_5 !== undefined
private readonly _hasColoAccess = process.env.OKX_API_COLO !== undefined

constructor(
private readonly _exchange: Exchange,
{ allowPublicBooksChannel }: { allowPublicBooksChannel: boolean }
{ allowPublicBooksChannel, validateSequence }: OkexV5BookChangeMapperOptions
) {
this._channelName = this._getBooksChannelName(allowPublicBooksChannel)
this.validateSequenceContinuity = validateSequence
}

canHandle(message: any) {
Expand All @@ -203,10 +226,6 @@ class OkexV5BookChangeMapper implements Mapper<OKEX_EXCHANGES, BookChange> {
return message.arg.channel === this._channelName
}

private _hasCredentials = process.env.OKX_API_KEY !== undefined
private _hasVip5Access = process.env.OKX_API_VIP_5 !== undefined
private _hasColoAccess = process.env.OKX_API_COLO !== undefined

private _getBooksChannelName(allowPublicBooksChannel: boolean) {
if (allowPublicBooksChannel === false) {
// historical data always uses books-l2-tbt
Expand Down Expand Up @@ -241,6 +260,10 @@ class OkexV5BookChangeMapper implements Mapper<OKEX_EXCHANGES, BookChange> {

*map(okexDepthDataMessage: OkexV5BookMessage, localTimestamp: Date): IterableIterator<BookChange> {
for (const message of okexDepthDataMessage.data) {
if (this.validateSequenceContinuity) {
this.validateSequence(okexDepthDataMessage, message)
}

if (okexDepthDataMessage.action === 'update' && message.bids.length === 0 && message.asks.length === 0) {
continue
}
Expand All @@ -263,6 +286,36 @@ class OkexV5BookChangeMapper implements Mapper<OKEX_EXCHANGES, BookChange> {
}
}
}

private validateSequence(okexDepthDataMessage: OkexV5BookMessage, message: OkexV5BookData) {
if (typeof message.seqId !== 'number') {
throw new Error(`OKX order book message is missing seqId, symbol: ${okexDepthDataMessage.arg.instId}`)
}

const symbol = okexDepthDataMessage.arg.instId
if (okexDepthDataMessage.action === 'snapshot') {
this.lastSequenceIdBySymbol.set(symbol, message.seqId)
return
}

if (typeof message.prevSeqId !== 'number') {
throw new Error(`OKX order book update is missing prevSeqId, symbol: ${symbol}, seqId: ${message.seqId}`)
}

const lastSequenceId = this.lastSequenceIdBySymbol.get(symbol)
if (lastSequenceId !== undefined && message.prevSeqId !== lastSequenceId) {
throw new Error(
`OKX order book sequence gap, symbol: ${symbol}, previous seqId: ${lastSequenceId}, update prevSeqId: ${message.prevSeqId}, update seqId: ${message.seqId}`
)
}

this.lastSequenceIdBySymbol.set(symbol, message.seqId)
}
}

type OkexV5BookChangeMapperOptions = {
allowPublicBooksChannel: boolean
validateSequence: boolean
}

class OkexV5BookTickerMapper implements Mapper<OKEX_EXCHANGES, BookTicker> {
Expand Down Expand Up @@ -740,22 +793,24 @@ type OkexV5BookLevel = [string, string, string, string]

type OkexV5BookMessage =
| {
arg: { channel: 'books-l2-tbt'; instId: string }
arg: { channel: 'books-l2-tbt' | 'books50-l2-tbt' | 'books'; instId: string }
action: 'snapshot'
data: [
{
asks: OkexV5BookLevel[]
bids: OkexV5BookLevel[]
ts: string
}
]
data: OkexV5BookData[]
}
| {
arg: { channel: 'books-l2-tbt'; instId: string }
arg: { channel: 'books-l2-tbt' | 'books50-l2-tbt' | 'books'; instId: string }
action: 'update'
data: [{ asks: OkexV5BookLevel[]; bids: OkexV5BookLevel[]; ts: string }]
data: OkexV5BookData[]
}

type OkexV5BookData = {
asks: OkexV5BookLevel[]
bids: OkexV5BookLevel[]
ts: string
seqId?: number
prevSeqId?: number
}

type OkexV5TickerMessage = {
arg: { channel: 'tickers'; instId: string }
data: [
Expand Down
171 changes: 171 additions & 0 deletions test/mappers.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1189,6 +1189,177 @@ describe('mappers', () => {
}
})

test('map okex public books validates seqId continuity', () => {
const localTimestamp = new Date()
const okexV5Mapper = createMapper('okex', localTimestamp)

expect(
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'snapshot',
data: [{ asks: [['2', '1', '0', '1']], bids: [['1', '1', '0', '1']], ts: '1779321600000', seqId: 100, prevSeqId: -1 }]
},
localTimestamp
)
).toEqual([
{
type: 'book_change',
symbol: 'BTC-USDT',
exchange: 'okex',
isSnapshot: true,
bids: [{ price: 1, amount: 1 }],
asks: [{ price: 2, amount: 1 }],
timestamp: new Date('2026-05-21T00:00:00.000Z'),
localTimestamp
}
])

expect(
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'update',
data: [{ asks: [['2', '0', '0', '1']], bids: [], ts: '1779321600001', seqId: 101, prevSeqId: 100 }]
},
localTimestamp
)
).toEqual([
{
type: 'book_change',
symbol: 'BTC-USDT',
exchange: 'okex',
isSnapshot: false,
bids: [],
asks: [{ price: 2, amount: 0 }],
timestamp: new Date('2026-05-21T00:00:00.001Z'),
localTimestamp
}
])
})

test('map okex public books advances sequence for empty updates', () => {
const localTimestamp = new Date()
const okexV5Mapper = createMapper('okex', localTimestamp)

okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'snapshot',
data: [{ asks: [], bids: [], ts: '1779321600000', seqId: 100, prevSeqId: -1 }]
},
localTimestamp
)

expect(
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'update',
data: [{ asks: [], bids: [], ts: '1779321600001', seqId: 101, prevSeqId: 100 }]
},
localTimestamp
)
).toEqual([])

expect(() =>
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'update',
data: [{ asks: [['2', '1', '0', '1']], bids: [], ts: '1779321600002', seqId: 102, prevSeqId: 101 }]
},
localTimestamp
)
).not.toThrow()
})

test('map okex public books throws on seqId gap', () => {
const localTimestamp = new Date()
const okexV5Mapper = createMapper('okex', localTimestamp)

okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'snapshot',
data: [{ asks: [], bids: [], ts: '1779321600000', seqId: 100, prevSeqId: -1 }]
},
localTimestamp
)

expect(() =>
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'update',
data: [{ asks: [['2', '1', '0', '1']], bids: [], ts: '1779321600001', seqId: 102, prevSeqId: 99 }]
},
localTimestamp
)
).toThrow('OKX order book sequence gap')
})

test('map okex historical public books does not enforce seqId continuity', () => {
const okexV5Mapper = createMapper('okex', new Date('2026-05-21T00:00:00.000Z'))
const localTimestamp = new Date('2026-05-21T00:00:00.000Z')

okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'snapshot',
data: [{ asks: [], bids: [], ts: '1779321600000', seqId: 100, prevSeqId: -1 }]
},
localTimestamp
)

expect(() =>
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'update',
data: [{ asks: [['2', '1', '0', '1']], bids: [], ts: '1779321600001', seqId: 102, prevSeqId: 99 }]
},
localTimestamp
)
).not.toThrow()
})

test('map okex public books requires sequence fields', () => {
const localTimestamp = new Date()
const okexV5Mapper = createMapper('okex', localTimestamp)

expect(() =>
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'BTC-USDT' },
action: 'snapshot',
data: [{ asks: [], bids: [], ts: '1779321600000' }]
},
localTimestamp
)
).toThrow('OKX order book message is missing seqId')

okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'ETH-USDT' },
action: 'snapshot',
data: [{ asks: [], bids: [], ts: '1779321600000', seqId: 100, prevSeqId: -1 }]
},
localTimestamp
)

expect(() =>
okexV5Mapper.map(
{
arg: { channel: 'books', instId: 'ETH-USDT' },
action: 'update',
data: [{ asks: [['2', '1', '0', '1']], bids: [], ts: '1779321600001', seqId: 101 }]
},
localTimestamp
)
).toThrow('OKX order book update is missing prevSeqId')
})

test('map okex-futures messages', () => {
const messages = [
{
Expand Down