diff --git a/nodejs/FlightRadarAPI/api.js b/nodejs/FlightRadarAPI/api.js index 19e92a6..6d50ebf 100644 --- a/nodejs/FlightRadarAPI/api.js +++ b/nodejs/FlightRadarAPI/api.js @@ -5,7 +5,7 @@ const Flight = require("./entities/flight"); const FlightTrackerConfig = require("./flightTrackerConfig"); const { AirportNotFoundError, LoginError } = require("./errors"); const { isNumeric, radians, rad2deg } = require("./util"); -const { parseAirlinesHtml, parseAirportsHtml } = require("./parsers"); +const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("./parsers"); /** @@ -198,23 +198,34 @@ class FlightRadar24API { } /** - * Return a list with all airports for specified countries. + * Return a list with all airports, optionally narrowed to some countries. * - * @param {Array} countries - Array of country names from Countries enum + * @param {Iterable|string} [countries] - Country names from the Countries enum, + * as any iterable or a single value; every country when omitted * @return {Promise>} */ async getAirports(countries) { - const airports = []; - // Use stateless requests for the fan-out so per-country `Set-Cookie` - // responses do not race onto the shared session jar. - await mapConcurrent(countries, this.maxWorkers, async (countryName) => { - const countryHref = Core.airportsDataUrl + "/" + countryName; - const { content } = await this.__client.requestStandalone( - countryHref, { headers: Core.htmlHeaders, timeout: this.timeout }, - ); - airports.push(...parseAirportsHtml(content, countryHref)); - }); - return airports; + // Any iterable, matching the Python port. Array.from would turn a + // non-iterable into [], answering a mistake with an empty result where + // Python raises. + let wanted = null; + + if (typeof countries === "string") { + wanted = [countries]; + } + else if (countries != null) { + if (typeof countries[Symbol.iterator] !== "function") { + throw new TypeError("getAirports expects a country name or an iterable of country names."); + } + wanted = Array.from(countries); + } + + if (wanted && wanted.length === 0) return []; + + const { content } = await this.__client.request( + Core.airportsJsonUrl, { headers: Core.jsonHeaders, timeout: this.timeout }, + ); + return parseAirportsJson(content, wanted); } /** @@ -315,7 +326,11 @@ class FlightRadar24API { * @return {Promise<[object, string] | null>} */ async getCountryFlag(country) { - const slug = country.toLowerCase().replaceAll(" ", "-"); + // Same slugifier as the feed, which spells some names "Myanmar (Burma)". + const slug = countryToSlug(country); + + if (!slug) return null; + const flagUrl = Core.countryFlagUrl(slug); const headers = { ...Core.imageHeaders }; diff --git a/nodejs/FlightRadarAPI/core.js b/nodejs/FlightRadarAPI/core.js index cc8707d..638362c 100644 --- a/nodejs/FlightRadarAPI/core.js +++ b/nodejs/FlightRadarAPI/core.js @@ -47,6 +47,8 @@ const Core = { apiAirportDataUrl: `${API_FR24_BASE}/airport.json`, airportDataUrl: (code) => `${FR24_BASE}/airports/traffic-stats/?airport=${code}`, airportsDataUrl: `${FR24_BASE}/data/airports`, + // Every airport in one payload; the per-country pages above are client-rendered. + airportsJsonUrl: `${FR24_BASE}/_json/airports.php`, airlinesDataUrl: `${FR24_BASE}/data/airlines`, diff --git a/nodejs/FlightRadarAPI/index.d.ts b/nodejs/FlightRadarAPI/index.d.ts index 2be3c99..0ed2c1a 100644 --- a/nodejs/FlightRadarAPI/index.d.ts +++ b/nodejs/FlightRadarAPI/index.d.ts @@ -28,7 +28,7 @@ export class APIClient { request(url: string, options?: object): Promise<{content: any; statusCode: number; cookies: Record}>; /** * Make a stateless request that bypasses the shared cookie jar. Safe to - * call from concurrent fan-outs (e.g. `getAirports`). + * call from concurrent fan-outs. */ requestStandalone(url: string, options?: object): Promise<{content: any; statusCode: number; cookies: Record}>; getCookie(name: string): string | undefined; @@ -114,11 +114,12 @@ export class FlightRadar24API { getAirportDisruptions(): Promise; /** - * Return a list with all airports for specified countries. + * Return a list with all airports, optionally narrowed to some countries. * - * @param {string[]} countries - Array of country names + * @param {Iterable|string} [countries] - Country names, as any iterable + * or a single value; every country when omitted */ - getAirports(countries: string[]): Promise; + getAirports(countries?: Iterable | string): Promise; /** * Return the bookmarks from the FlightRadar24 account. @@ -311,8 +312,8 @@ export class Entity { * Airport representation. */ export class Airport extends Entity { - latitude: number; - longitude: number; + latitude: number | null; + longitude: number | null; altitude: number | null; name: string; icao: string; diff --git a/nodejs/FlightRadarAPI/parsers.js b/nodejs/FlightRadarAPI/parsers.js index e7856a9..79a16dd 100644 --- a/nodejs/FlightRadarAPI/parsers.js +++ b/nodejs/FlightRadarAPI/parsers.js @@ -71,89 +71,163 @@ function parseAirlinesHtml(html) { } /** - * Parse the airports listing HTML page for a country into a list of Airport instances. + * Slugify a country name the way FR24 spells it in its data page URLs, so feed + * rows can be matched against the Countries enum. * - * @param {string|Buffer} html - * @param {string} countryHref - Full URL used to fetch the page (used to derive the display name) - * @return {Array} + * @param {string} country - Country name as spelled by FR24 (e.g. "United States") + * @return {string} URL-friendly form matching the Countries enum (e.g. "united-states") */ -function parseAirportsHtml(html, countryHref) { - const root = parse(typeof html === "string" ? html : html.toString()); - const tbody = root.querySelector("tbody"); +function countryToSlug(country) { + // Diacritics stripped so a future "Curaçao" still matches "curacao". + // Punctuation becomes a hyphen rather than being deleted: FR24's own assets + // are named that way, e.g. flags-small/cote-d-ivoire.svg. + return String(country ?? "") + .normalize("NFKD").replace(/[\u0300-\u036f]/g, "") + .toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, ""); +} - if (!tbody) { - console.warn(`parseAirportsHtml: no for ${countryHref} — FR24 page layout may have changed.`); - return []; - } +// ASCII decimals only, so both ports accept and reject the same strings. +const NUMERIC_PATTERN = /^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?$/; - const countryDisplayName = countryHref.split("/").pop().replace(/-/g, " ") - .split(" ").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" "); +// trim() and str.strip() disagree on U+FEFF and U+001C-U+001F: pick one set. +const SURROUNDING_SPACE = /^[ \t\n\r\f\v]+|[ \t\n\r\f\v]+$/g; - const airports = []; +/** + * Coerce a numeric feed field into a number, or null when it is unusable. + * + * An unusable coordinate must not become 0: that would place the airport in the + * Gulf of Guinea instead of marking its position as unknown. + * + * @param {*} value + * @return {number|null} + */ +function toNumber(value) { + if (typeof value === "number") return Number.isFinite(value) ? value : null; + // Not stringified: String([43]) would pass as 43. + if (typeof value !== "string") return null; - for (const tr of tbody.querySelectorAll("tr")) { - const aElement = tr.querySelector("a[data-iata][data-lat][data-lon]"); + const text = value.replace(SURROUNDING_SPACE, ""); - if (!aElement) continue; + return NUMERIC_PATTERN.test(text) ? toNumber(Number(text)) : null; +} + +/** + * Keep a text field as a string, or "" when the feed sends anything else. + * + * Airport declares these as strings, and callers treat them as such — + * `getCountryFlag(airport.country)` would happily slugify an object into + * "object-object" and request that flag. + * + * @param {*} value + * @return {string} + */ +function toStringField(value) { + return typeof value === "string" ? value : ""; +} + +/** + * Decode a response body into text, or null when it arrived already parsed. + * + * `request()` returns an ArrayBuffer whenever the response carries an + * unexpected content-type, so a perfectly good JSON body can land here as raw + * bytes rather than as an object. + * + * @param {*} payload + * @return {string|null} + */ +function toText(payload) { + if (typeof payload === "string") return payload; + if (Buffer.isBuffer(payload)) return payload.toString(); + if (payload instanceof ArrayBuffer) return Buffer.from(payload).toString(); + if (ArrayBuffer.isView(payload)) return Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength).toString(); + return null; +} - let icao = ""; - let iata = aElement.getAttribute("data-iata") || ""; - const latitude = aElement.getAttribute("data-lat") || ""; - const longitude = aElement.getAttribute("data-lon") || ""; - let namePart = aElement.text.trim(); +/** + * Parse the airports JSON feed into a list of Airport instances. + * + * @param {object|string|Buffer} payload - Body of Core.airportsJsonUrl + * @param {Array} [countries] - Country slugs from the Countries enum; every airport is kept when omitted + * @return {Array} + */ +function parseAirportsJson(payload, countries = null) { + let data = payload; + const text = toText(payload); - const smallElement = aElement.querySelector("small"); + if (text !== null) { + try { + data = JSON.parse(text); + } + catch { + console.warn("parseAirportsJson: response is not valid JSON — FR24 feed may have changed."); + return []; + } + } - if (smallElement) { - const smallText = smallElement.text.trim(); - const codesText = smallText.replace(/^\(/, "").replace(/\)$/, "").trim(); - namePart = namePart.replace(smallText, "").replace(/\(\)/, "").trim(); + const rows = data && Array.isArray(data["rows"]) ? data["rows"] : null; - if (codesText.includes("/")) { - const codes = codesText.split("/"); - const code1 = codes[0].trim(); - const code2 = codes[1].trim(); + if (!rows) { + console.warn("parseAirportsJson: no \"rows\" array in response — FR24 feed may have changed."); + return []; + } - if (code1.length === 3 && code2.length === 4) { - iata = code1; - icao = code2; - } - else if (code1.length === 4 && code2.length === 3) { - iata = code2; - icao = code1; - } - } - else if (codesText.length === 3) { - iata = codesText; - } - else if (codesText.length === 4) { - icao = codesText; - } + const wanted = countries ? new Set(countries.map(countryToSlug)) : null; + const matched = new Set(); + const unpositioned = []; + const airports = []; + + for (const row of rows) { + // Arrays are objects in JS; Python's isinstance(row, dict) drops them. + if (!row || typeof row !== "object" || Array.isArray(row)) continue; + + // Slugified only when filtering: it is otherwise unused, and this loop + // runs over every airport in the feed. + if (wanted) { + const slug = countryToSlug(row["country"]); + + if (!wanted.has(slug)) continue; + matched.add(slug); } - let latNum = latitude ? parseFloat(latitude) : null; - let lonNum = longitude ? parseFloat(longitude) : null; - if (Number.isNaN(latNum) || Number.isNaN(lonNum)) { - console.warn( - `parseAirportsHtml: invalid coordinates for airport "${namePart}" ` + - `(lat=${latitude}, lon=${longitude}) — skipping position.`, - ); - latNum = null; - lonNum = null; + let latitude = toNumber(row["lat"]); + let longitude = toNumber(row["lon"]); + + // One bad coordinate drops both: half a position reads as located. + if (latitude === null || longitude === null) { + latitude = null; + longitude = null; + unpositioned.push(toStringField(row["name"])); } airports.push(new Airport({ - "name": namePart, - "icao": icao, - "iata": iata, - "lat": latNum, - "lon": lonNum, - "alt": null, - "country": countryDisplayName, + "name": toStringField(row["name"]), + "icao": toStringField(row["icao"]), + "iata": toStringField(row["iata"]), + "lat": latitude, + "lon": longitude, + "alt": toNumber(row["alt"]), + "country": toStringField(row["country"]), })); } + // One line, not one per row: the feed carries every airport. + if (unpositioned.length > 0) { + const sample = unpositioned.slice(0, 3).map((name) => `"${name}"`).join(", "); + console.warn( + `parseAirportsJson: ${unpositioned.length} airport(s) had unusable coordinates ` + + `and carry no position (e.g. ${sample}).`, + ); + } + + if (wanted) { + const missing = [...wanted].filter((slug) => !matched.has(slug)); + + if (missing.length > 0) { + console.warn(`parseAirportsJson: no airports found for ${missing.join(", ")} — check the Countries enum.`); + } + } + return airports; } -module.exports = { parseAirlinesHtml, parseAirportsHtml }; +module.exports = { parseAirlinesHtml, parseAirportsJson, countryToSlug }; diff --git a/nodejs/FlightRadarAPI/request.js b/nodejs/FlightRadarAPI/request.js index 4e9c505..94a07d9 100644 --- a/nodejs/FlightRadarAPI/request.js +++ b/nodejs/FlightRadarAPI/request.js @@ -364,9 +364,9 @@ class APIClient { /** * Make a stateless request that does not touch the shared cookie jar. * - * Safe to call from concurrent fan-outs (e.g. `getAirports` issuing one - * request per country). The TLS dispatcher is still reused so the - * impersonation profile stays consistent with the session. + * Safe to call from concurrent fan-outs, where per-response `Set-Cookie` + * headers would otherwise race onto the shared jar. The TLS dispatcher is + * still reused so the impersonation profile stays consistent with the session. * * @param {string} url * @param {object} [options={}] diff --git a/nodejs/README.md b/nodejs/README.md index 55f9c4e..3abf5b9 100644 --- a/nodejs/README.md +++ b/nodejs/README.md @@ -31,10 +31,13 @@ const frApi = new FlightRadar24API(); let flights = await frApi.getFlights(...); // Returns a list of Flight objects ``` -**Getting airports list (requires country selection):** +**Getting airports list:** ```javascript // Get airports from specific countries let airports = await frApi.getAirports([Countries.BRAZIL, Countries.UNITED_STATES]); // Returns a list of Airport objects + +// Omit the countries to get every airport +let allAirports = await frApi.getAirports(); ``` **Getting airlines list:** diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index b39a1b8..a28f5c9 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "flightradarapi", - "version": "1.5.2", + "version": "1.5.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "flightradarapi", - "version": "1.5.2", + "version": "1.5.3", "license": "MIT", "dependencies": { "node-html-parser": "^6.1.13", diff --git a/nodejs/package.json b/nodejs/package.json index b4fb9a3..9ff4cac 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "flightradarapi", - "version": "1.5.2", + "version": "1.5.3", "description": "SDK for FlightRadar24", "main": "./FlightRadarAPI/index.js", "types": "./FlightRadarAPI/index.d.ts", diff --git a/nodejs/tests/fixtures/airports.json b/nodejs/tests/fixtures/airports.json new file mode 100644 index 0000000..924f9db --- /dev/null +++ b/nodejs/tests/fixtures/airports.json @@ -0,0 +1,77 @@ +{ + "version": "1786010717", + "rows": [ + { + "name": "Sao Paulo Guarulhos International Airport", + "iata": "GRU", + "icao": "SBGR", + "lat": -23.429991, + "lon": -46.4674, + "country": "Brazil", + "alt": 2436 + }, + { + "name": "Rio de Janeiro Galeao International Airport", + "iata": "GIG", + "icao": "SBGL", + "lat": -22.805696, + "lon": -43.25523, + "country": "Brazil", + "alt": 22 + }, + { + "name": "Sao Paulo Congonhas Airport", + "iata": "CGH", + "icao": "SBSP", + "lat": -23.627348, + "lon": -46.655987, + "country": "Brazil", + "alt": 2614 + }, + { + "name": "Los Angeles International Airport", + "iata": "LAX", + "icao": "KLAX", + "lat": 33.94252, + "lon": -118.406998, + "country": "United States", + "alt": 125 + }, + { + "name": "New York John F. Kennedy International Airport", + "iata": "JFK", + "icao": "KJFK", + "lat": 40.639751, + "lon": -73.7789, + "country": "United States", + "alt": 14 + }, + { + "name": "A Coruna Airport", + "iata": "LCG", + "icao": "LECO", + "lat": 43.302059, + "lon": -8.37725, + "country": "Spain", + "alt": 326 + }, + { + "name": "Ann Airport", + "iata": "VBA", + "icao": "VYAN", + "lat": 19.770643, + "lon": 94.026405, + "country": "Myanmar (Burma)", + "alt": "43" + }, + { + "name": "Broken Coords Airport", + "iata": "BAD", + "icao": "SBAD", + "lat": "n/a", + "lon": "", + "country": "Brazil", + "alt": 0 + } + ] +} diff --git a/nodejs/tests/fixtures/airports_brazil.html b/nodejs/tests/fixtures/airports_brazil.html deleted file mode 100644 index dc62403..0000000 --- a/nodejs/tests/fixtures/airports_brazil.html +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - -
- São Paulo / Guarulhos (GRU/SBGR) -
- Rio de Janeiro / Galeão (GIG/SBGL) -
- São Paulo / Congonhas (CGH) -
- Bad Coords Airport (BAD/SBBD) -
placeholder
- diff --git a/nodejs/tests/index.test-d.ts b/nodejs/tests/index.test-d.ts index c5aaaa1..f7641a4 100644 --- a/nodejs/tests/index.test-d.ts +++ b/nodejs/tests/index.test-d.ts @@ -26,6 +26,9 @@ expectType>(api.getAirportDetails("ATL", 10, 1)); // getAirports expectType>(api.getAirports(["Brazil"])); +expectType>(api.getAirports("Brazil")); +expectType>(api.getAirports(new Set(["Brazil"]))); +expectType>(api.getAirports()); // getAirlines expectType>>(api.getAirlines()); diff --git a/nodejs/tests/testApi.js b/nodejs/tests/testApi.js index 0767c8b..c20ddaa 100644 --- a/nodejs/tests/testApi.js +++ b/nodejs/tests/testApi.js @@ -1,4 +1,5 @@ const { FlightRadar24API, Flight, Entity, FlightTrackerConfig, Countries, version } = require(".."); +const { countryToSlug } = require("../FlightRadarAPI/parsers"); const expect = require("chai").expect; @@ -50,6 +51,35 @@ describe("Testing FlightRadarAPI version " + version, function() { const results = await frApi.getAirports(countries); expect(results.length).to.be.above(expected - 1); }); + + let everyAirport; + before(async function() { + everyAirport = await frApi.getAirports(); + }); + + it("Expected every airport when no country is given, each with a flag.", async function() { + expect(everyAirport.length).to.be.above(expected); + expect(new Set(everyAirport.map((airport) => airport.country)).size).to.be.above(countries.length); + + // Discovered, not hard-coded: FR24 spells some names "Myanmar (Burma)". + // Sorted so the pick does not depend on the order the feed happens to + // list airports in. + const punctuated = everyAirport.map((airport) => airport.country).filter((name) => /[^a-z ]/i.test(name)); + const tricky = punctuated.sort()[0] ?? everyAirport[0].country; + + expect(await frApi.getCountryFlag(tricky), tricky).to.not.equal(null); + }); + + it("Expected every Countries value to name a country the feed knows.", async function() { + // The enum's URL slugs and the feed's display names are two FR24 + // vocabularies; a rename silently empties that country's filter. The + // reverse direction is not asserted: a country FR24 adds is a gap in + // the enum, not a regression. + const feed = new Set(everyAirport.map((airport) => countryToSlug(airport.country))); + const absent = Object.values(Countries).filter((value) => !feed.has(value)); + + expect(absent, "enum values absent from the feed").to.deep.equal([]); + }); }); describe("Getting Zones", function() { @@ -126,11 +156,13 @@ describe("Testing FlightRadarAPI version " + version, function() { const flights = await frApi.getFlights(null, bounds); for (const flight of flights) { - expect(flight.latitude).to.be.below(zone["tl_y"]); - expect(flight.latitude).to.be.above(zone["br_y"]); + // Inclusive, like the Python port: FR24 returns flights sitting + // exactly on the boundary of the requested box. + expect(flight.latitude).to.be.at.most(zone["tl_y"]); + expect(flight.latitude).to.be.at.least(zone["br_y"]); - expect(flight.longitude).to.be.below(zone["br_x"]); - expect(flight.longitude).to.be.above(zone["tl_x"]); + expect(flight.longitude).to.be.at.most(zone["br_x"]); + expect(flight.longitude).to.be.at.least(zone["tl_x"]); } expect(flights.length).to.be.above(expected - 1); } diff --git a/nodejs/tests/testParsersOffline.js b/nodejs/tests/testParsersOffline.js index 2e71b15..0326870 100644 --- a/nodejs/tests/testParsersOffline.js +++ b/nodejs/tests/testParsersOffline.js @@ -1,21 +1,34 @@ /** - * Offline parser tests against bundled HTML fixtures. + * Offline parser tests against bundled fixtures. * * These tests run with no network access and exist specifically so that PRs * can be gated on the parser logic without depending on FR24 being reachable - * or its HTML layout being stable. When FR24 changes the page structure, - * update the fixtures (re-saving a real page is fine) — the assertions here - * guard the parser's invariants, not byte-for-byte equality with production. + * or its payloads being stable. When FR24 changes a page or feed, update the + * fixtures (re-saving a real response is fine) — the assertions here guard the + * parser's invariants, not byte-for-byte equality with production. */ const fs = require("fs"); const path = require("path"); const expect = require("chai").expect; -const { parseAirlinesHtml, parseAirportsHtml } = require("../FlightRadarAPI/parsers"); +const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("../FlightRadarAPI/parsers"); +const { Countries } = require("../FlightRadarAPI/core"); const FIXTURES = path.join(__dirname, "fixtures"); const load = (name) => fs.readFileSync(path.join(FIXTURES, name), "utf-8"); +// Keep in step with the same list in python/tests/test_parsers_offline.py. +const NUMERIC_CASES = [ + ["whitespace", " ", null], + ["an array holding a number", [43], null], + ["a coordinate with a hemisphere suffix", "43.30 N", null], + ["an exponent overflowing a double", "1e999", null], + ["400 plain digits", "1".repeat(400), null], + ["a whole number as a string", "2436", 2436], + ["a negative decimal", "-23.4", -23.4], + ["a genuine zero", 0, 0], +]; + describe("parseAirlinesHtml (offline)", function() { let airlines; @@ -63,10 +76,10 @@ describe("parseAirlinesHtml (offline)", function() { }); -describe("parseAirportsHtml (offline)", function() { +describe("parseAirportsJson (offline)", function() { let airports; before(function() { - airports = parseAirportsHtml(load("airports_brazil.html"), "/data/airports/brazil"); + airports = parseAirportsJson(load("airports.json")); }); it("extracts iata, icao, name, country and position", function() { @@ -75,13 +88,30 @@ describe("parseAirportsHtml (offline)", function() { expect(byIata["GIG"]).to.exist; expect(byIata["GRU"].icao).to.equal("SBGR"); expect(byIata["GRU"].country).to.equal("Brazil"); - expect(byIata["GRU"].latitude).to.be.closeTo(-23.4356, 1e-6); - expect(byIata["GRU"].longitude).to.be.closeTo(-46.4731, 1e-6); + expect(byIata["GRU"].latitude).to.be.closeTo(-23.429991, 1e-6); + expect(byIata["GRU"].longitude).to.be.closeTo(-46.4674, 1e-6); + expect(byIata["GRU"].altitude).to.equal(2436); + }); + + it("keeps every country when no filter is given", function() { + const countries = new Set(airports.map((a) => a.country)); + expect(countries).to.include("Brazil"); + expect(countries).to.include("United States"); + expect(countries).to.include("Spain"); + }); + + it("filters by the slugs from the Countries enum", function() { + const filtered = parseAirportsJson(load("airports.json"), ["united-states"]); + expect(filtered.length).to.be.above(0); + for (const airport of filtered) { + expect(airport.country).to.equal("United States"); + } }); - it("handles airports whose small tag has only one code", function() { - const cgh = airports.find((a) => a.iata === "CGH"); - expect(cgh.icao).to.equal(""); + it("accepts several countries at once", function() { + const filtered = parseAirportsJson(load("airports.json"), ["brazil", "spain"]); + const countries = new Set(filtered.map((a) => a.country)); + expect([...countries].sort()).to.deep.equal(["Brazil", "Spain"]); }); it("invalid coordinates become null (regression test for 0,0 fallback)", function() { @@ -90,14 +120,136 @@ describe("parseAirportsHtml (offline)", function() { expect(bad.longitude).to.equal(null); }); - it("derives the display country from the URL slug", function() { - const usAirports = parseAirportsHtml(load("airports_brazil.html"), "/data/airports/united-states"); - for (const a of usAirports) { - expect(a.country).to.equal("United States"); + it("keeps FR24's own country spelling and still slugifies it for URLs", function() { + const ann = airports.find((airport) => airport.iata === "VBA"); + + expect(ann.country).to.equal("Myanmar (Burma)"); + expect(countryToSlug(ann.country)).to.equal("myanmar-burma"); + expect(ann.altitude).to.equal(43); + }); + + it("matches a filter written with either spelling of the country", function() { + for (const slug of ["myanmar-burma", "Myanmar (Burma)"]) { + expect(parseAirportsJson(load("airports.json"), [slug]).length).to.equal(1); } }); - it("returns an empty list for empty input", function() { - expect(parseAirportsHtml("", "/data/airports/brazil")).to.deep.equal([]); + it("rejects numeric-looking junk instead of truncating it", function() { + const payload = JSON.stringify({ rows: [{ + name: "Junk Airport", iata: "JNK", icao: "JJNK", + lat: "43.30 N", lon: -8.37725, country: "Spain", alt: "-1", + }] }); + const [airport] = parseAirportsJson(payload); + + // One bad coordinate drops both; altitude is independent and survives. + expect(airport.latitude).to.equal(null); + expect(airport.longitude).to.equal(null); + expect(airport.altitude).to.equal(-1); + }); + + it("agrees with the Python port on Unicode digits and stray control bytes", function() { + // Each port used to accept what the other rejected. + const read = (value) => { + const payload = JSON.stringify({ rows: [{ + name: "X", iata: "XXX", icao: "XXXX", country: "Spain", lat: value, lon: value, alt: value, + }] }); + return parseAirportsJson(payload)[0].altitude; + }; + + expect(read("\u0664\u0663"), "Arabic-Indic digits").to.equal(null); + expect(read("\ufeff43"), "leading U+FEFF").to.equal(null); + expect(read("\u001c43"), "leading U+001C").to.equal(null); + expect(read(" 43 "), "plain spaces still trimmed").to.equal(43); + }); + + it("returns an empty list for an unknown country", function() { + expect(parseAirportsJson(load("airports.json"), ["atlantis"])).to.deep.equal([]); + }); + + it("survives numbers written raw in the JSON that no double can hold", function() { + // Written as text: a JS literal is already rounded before it reaches here. + for (const literal of ["1".repeat(400), "9007199254740993", "-1".concat("1".repeat(399))]) { + const payload = `{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":${literal},"lon":1,"alt":2}]}`; + const [airport] = parseAirportsJson(payload); + + expect(airport.latitude, literal.slice(0, 12)).to.satisfy( + (latitude) => latitude === null || Number.isFinite(latitude), + ); + } + }); + + it("keeps text fields as strings whatever the feed sends", function() { + // Anything else breaks getCountryFlag(airport.country). + for (const literal of ["null", "0", "false", "true", "[]", "{}", "123"]) { + const payload = `{"rows":[{"name":${literal},"iata":${literal},"icao":${literal},` + + `"country":${literal},"lat":1,"lon":2,"alt":3}]}`; + const [airport] = parseAirportsJson(payload); + + expect([airport.name, airport.iata, airport.icao, airport.country], literal) + .to.deep.equal(["", "", "", ""]); + } + }); + + it("parses a body that arrived as raw bytes", function() { + // request() returns an ArrayBuffer on an unexpected content-type. + const json = load("airports.json"); + + for (const payload of [Buffer.from(json), new TextEncoder().encode(json), new TextEncoder().encode(json).buffer]) { + expect(parseAirportsJson(payload).length, payload.constructor.name).to.equal(airports.length); + } + }); + + it("skips rows that are not objects, as the Python port does", function() { + const payload = "{\"rows\":[[1,2,3],\"x\",7,null," + + "{\"name\":\"Real\",\"iata\":\"RRR\",\"icao\":\"RRRR\",\"country\":\"Spain\",\"lat\":1,\"lon\":2,\"alt\":3}]}"; + + expect(parseAirportsJson(payload).map((airport) => airport.iata)).to.deep.equal(["RRR"]); + }); + + it("returns an empty list when the feed has no rows", function() { + expect(parseAirportsJson("")).to.deep.equal([]); + expect(parseAirportsJson("{}")).to.deep.equal([]); + expect(parseAirportsJson("not json")).to.deep.equal([]); + }); +}); + + +describe("parseAirportsJson numeric coercion", function() { + for (const [label, input, expected] of NUMERIC_CASES) { + it(`reads ${label} as ${JSON.stringify(expected)}`, function() { + const payload = JSON.stringify({ rows: [{ + name: "X", iata: "XXX", icao: "XXXX", country: "Spain", + lat: input, lon: input, alt: input, + }] }); + const [airport] = parseAirportsJson(payload); + + expect(airport.latitude, "latitude").to.equal(expected); + expect(airport.longitude, "longitude").to.equal(expected); + expect(airport.altitude, "altitude").to.equal(expected); + }); + } +}); + + +describe("countryToSlug (offline)", function() { + it("matches the spelling used by the Countries enum", function() { + expect(countryToSlug("United States")).to.equal("united-states"); + expect(countryToSlug("Democratic Republic Of The Congo")).to.equal("democratic-republic-of-the-congo"); + expect(countryToSlug("Curacao")).to.equal("curacao"); + expect(countryToSlug("Curaçao")).to.equal("curacao"); + expect(countryToSlug("")).to.equal(""); + }); + + it("takes a Countries value, which is what getAirports accepts", function() { + expect(countryToSlug(Countries.BRAZIL)).to.equal("brazil"); + expect(countryToSlug(Countries.MYANMAR_BURMA)).to.equal("myanmar-burma"); + }); + + it("strips the parentheses FR24 uses, keeping country flag URLs valid", function() { + // These 404'd while the slug was a plain space-to-hyphen replacement. + expect(countryToSlug("Myanmar (Burma)")).to.equal("myanmar-burma"); + expect(countryToSlug("Cocos (Keeling) Islands")).to.equal("cocos-keeling-islands"); + expect(countryToSlug("Falkland Islands (Malvinas)")).to.equal("falkland-islands-malvinas"); + expect(countryToSlug("Timor-Leste (East Timor)")).to.equal("timor-leste-east-timor"); }); }); diff --git a/python/FlightRadarAPI/__init__.py b/python/FlightRadarAPI/__init__.py index e7c008a..52e7635 100644 --- a/python/FlightRadarAPI/__init__.py +++ b/python/FlightRadarAPI/__init__.py @@ -12,7 +12,7 @@ """ __author__ = "Jean Loui Bernard Silva de Jesus" -__version__ = "1.5.2" +__version__ = "1.5.3" from .api import FlightRadar24API from .core import Countries diff --git a/python/FlightRadarAPI/api.py b/python/FlightRadarAPI/api.py index a097462..e440e8a 100644 --- a/python/FlightRadarAPI/api.py +++ b/python/FlightRadarAPI/api.py @@ -3,7 +3,7 @@ import dataclasses import math from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from urllib.parse import quote from .core import Core, Countries @@ -11,7 +11,7 @@ from .entities.flight import Flight from .errors import AirportNotFoundError, LoginError from .flight_tracker_config import FlightTrackerConfig -from .parsers import parse_airlines_html, parse_airports_html +from .parsers import country_to_slug, parse_airlines_html, parse_airports_json from .request import APIClient, RetryPolicy # Some FR24 live-feed backends answer 200 with a well-formed envelope but no @@ -187,21 +187,33 @@ def get_airport_disruptions(self) -> Dict: ) return response.get_json_content() - def get_airports(self, countries: List[Countries]) -> List[Airport]: + def get_airports( + self, countries: Optional[Union[Iterable[Union[Countries, str]], Countries, str]] = None, + ) -> List[Airport]: """ - Return a list with all airports for specified countries. + Return a list with all airports, optionally narrowed to some countries. - :param countries: List of country names from Countries enum. + :param countries: Country names from the Countries enum, or their slug strings, + as any iterable or a single value. Every country when omitted. """ - def _fetch(country): - href = Core.airports_data_url + "/" + country.value - response = self.__client.request_standalone(href, headers=Core.html_headers, timeout=self.timeout) - return parse_airports_html(response.get_bytes_content(), href) + wanted: Optional[List[Union[Countries, str]]] = None - with ThreadPoolExecutor(max_workers=self.max_workers) as executor: - results = executor.map(_fetch, countries) + if isinstance(countries, (Countries, str)): + wanted = [countries] + elif countries is not None: + # Materialised: a generator has no len() and is consumed once. + wanted = list(countries) - return [airport for result in results for airport in result] + if wanted is not None and len(wanted) == 0: + return [] + + response = self.__client.request( + Core.airports_json_url, headers=Core.json_headers, timeout=self.timeout, + ) + + # get_content(), not get_json_content(): an html body reaches the parser's + # guard; a malformed JSON body still raises. The parser slugifies `wanted`. + return parse_airports_json(response.get_content(), wanted) def get_bookmarks(self) -> Dict: """ @@ -292,7 +304,12 @@ def get_country_flag(self, country: str) -> Optional[Tuple[bytes, str]]: :param country: Country name """ - slug = country.lower().replace(" ", "-") + # Same slugifier as the feed, which spells some names "Myanmar (Burma)". + slug = country_to_slug(country) + + if not slug: + return None + flag_url = Core.country_flag_url.format(slug) headers = Core.image_headers.copy() diff --git a/python/FlightRadarAPI/core.py b/python/FlightRadarAPI/core.py index 79053d3..5948e6c 100644 --- a/python/FlightRadarAPI/core.py +++ b/python/FlightRadarAPI/core.py @@ -32,6 +32,9 @@ class Core: airport_data_url = flightradar_base_url + "/airports/traffic-stats/?airport={}" airports_data_url = flightradar_base_url + "/data/airports" + # Every airport in one payload; the per-country pages above are client-rendered. + airports_json_url = flightradar_base_url + "/_json/airports.php" + # Airlines data URL. airlines_data_url = flightradar_base_url + "/data/airlines" diff --git a/python/FlightRadarAPI/parsers.py b/python/FlightRadarAPI/parsers.py index 813d65f..77667ad 100644 --- a/python/FlightRadarAPI/parsers.py +++ b/python/FlightRadarAPI/parsers.py @@ -1,7 +1,12 @@ # -*- coding: utf-8 -*- +import json import logging -from typing import Dict, List, Optional +import math +import re +import unicodedata +from enum import Enum +from typing import Dict, Iterable, List, Optional, Union from bs4 import BeautifulSoup @@ -9,6 +14,15 @@ _logger = logging.getLogger(__name__) +# ASCII only: \d would otherwise match Unicode digits such as "٤٣". +NUMERIC_PATTERN = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$", re.ASCII) + +# str.strip() and trim() disagree on U+001C-U+001F and U+FEFF: pick one set. +SURROUNDING_SPACE = " \t\n\r\f\v" + +# Number.MAX_SAFE_INTEGER, the largest integer JavaScript holds exactly. +MAX_EXACT_INTEGER = 2 ** 53 - 1 + def parse_airlines_html(html: bytes) -> List[Dict]: """ @@ -70,74 +84,156 @@ def parse_airlines_html(html: bytes) -> List[Dict]: return airlines -def parse_airports_html(html: bytes, country_href: str) -> List[Airport]: +def country_to_slug(country: object) -> str: """ - Parse the airports listing HTML page for a country into a list of Airport instances. + Slugify a country name the way FR24 spells it in its data page URLs, so feed + rows can be matched against the Countries enum ("United States" -> "united-states"). """ - soup = BeautifulSoup(html, "html.parser") - tbody = soup.find("tbody") + # `.value` unwraps a Countries member, which str() would render as + # "Countries.BRAZIL". Diacritics are stripped so a future "Curaçao" still + # matches "curacao"; `is None` rather than truthiness because str(0 or "") is + # "" here while String(0 ?? "") is "0" in Node. + if isinstance(country, Enum): + country = country.value - if not tbody: - _logger.warning( - "parse_airports_html: no for %s — FR24 page layout may have changed.", - country_href, - ) - return [] + decomposed = unicodedata.normalize("NFKD", "" if country is None else str(country)) + ascii_only = "".join(char for char in decomposed if not unicodedata.combining(char)) - country_name = country_href.split("/")[-1].replace("-", " ").title() - airports = [] + # Punctuation becomes a hyphen rather than being deleted: FR24's own assets are + # named that way, e.g. flags-small/cote-d-ivoire.svg. + return re.sub(r"[^a-z0-9]+", "-", ascii_only.lower()).strip("-") - for tr in tbody.find_all("tr"): - a_elements = tr.find_all("a", attrs={"data-iata": True, "data-lat": True, "data-lon": True}) - if not a_elements: - continue +def _to_text(value: object) -> str: + """ + Keep a text field as a string, or "" when the feed sends anything else. - a_element = a_elements[0] + Airport documents these as strings and callers treat them as such: + get_country_flag(airport.country) would slugify a dict into "object-object" + and request that flag. + """ + return value if isinstance(value, str) else "" - icao = "" - iata = str(a_element.get("data-iata", "")).strip() - latitude = str(a_element.get("data-lat", "")).strip() - longitude = str(a_element.get("data-lon", "")).strip() - name_part = a_element.get_text(strip=True) - small_element = a_element.find("small") +def _to_number(value: object) -> Optional[Union[int, float]]: + """ + Coerce a numeric feed field into a number, or None when it is unusable. - if small_element: - codes_text = small_element.get_text(strip=True).lstrip("(").rstrip(")").strip() - name_part = name_part.replace(small_element.get_text(strip=True), "").replace("()", "").strip() + An unusable coordinate must not become 0.0: that would place the airport in + the Gulf of Guinea instead of marking its position as unknown. - if "/" in codes_text: - code1, code2 = (s.strip() for s in codes_text.split("/", maxsplit=1)) - if len(code1) == 3 and len(code2) == 4: - iata, icao = code1, code2 - elif len(code1) == 4 and len(code2) == 3: - iata, icao = code2, code1 - elif len(codes_text) == 3: - iata = codes_text - elif len(codes_text) == 4: - icao = codes_text + Plain whole numbers stay ints, so an altitude reads 2436 in both ports rather + than 2436.0 here -- the feed sends `alt` as an int for most rows and as a + string ("-1") for the rest. Anything written with a decimal point or an + exponent stays a float: "2436.0" and "1e3" are floats here and integral + Numbers in Node, equal in value but not in type. + """ + # bool is an int here, and str([43]) would invent a number. + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + + if isinstance(value, str): + stripped = value.strip(SURROUNDING_SPACE) + + if not NUMERIC_PATTERN.match(stripped): + return None + + text: Union[int, float, str] = stripped + else: + text = value + + # One conversion, so overflow is handled once: Python ints have no ceiling. + try: + number = float(text) + except OverflowError: + return None + + if not math.isfinite(number): + return None + + try: + exact = int(text) + except (ValueError, OverflowError): + return number - lat_float: Optional[float] - lon_float: Optional[float] + # int only when lossless (so -23.4 survives) and only where JavaScript is exact. + return exact if exact == number and abs(exact) <= MAX_EXACT_INTEGER else number + + +def parse_airports_json( + payload: Union[bytes, str, Dict], countries: Optional[Iterable[object]] = None, +) -> List[Airport]: + """ + Parse the airports JSON feed into a list of Airport instances. + + :param payload: Body of Core.airports_json_url. + :param countries: Country slugs, or Countries members. Every airport is kept when omitted. + """ + if isinstance(payload, (bytes, str)): try: - lat_float = float(latitude) if latitude else None - lon_float = float(longitude) if longitude else None + data = json.loads(payload) except ValueError: - _logger.warning( - "parse_airports_html: invalid coordinates for airport %r (lat=%r, lon=%r) — skipping position.", - name_part, latitude, longitude, - ) - lat_float, lon_float = None, None + _logger.warning("parse_airports_json: response is not valid JSON — FR24 feed may have changed.") + return [] + else: + data = payload + + rows = data.get("rows") if isinstance(data, dict) else None + + if not isinstance(rows, list): + _logger.warning('parse_airports_json: no "rows" array in response — FR24 feed may have changed.') + return [] + + wanted = {country_to_slug(country) for country in countries} if countries is not None else None + matched = set() + unpositioned = [] + airports = [] + + for row in rows: + if not isinstance(row, dict): + continue + + # Slugified only when filtering: it is otherwise unused, and this loop + # runs over every airport in the feed. + if wanted is not None: + slug = country_to_slug(row.get("country")) + + if slug not in wanted: + continue + matched.add(slug) + + latitude = _to_number(row.get("lat")) + longitude = _to_number(row.get("lon")) + + # One bad coordinate drops both: half a position reads as located. + if latitude is None or longitude is None: + latitude = longitude = None + unpositioned.append(_to_text(row.get("name"))) airports.append(Airport(basic_info={ - "name": name_part, - "icao": icao, - "iata": iata, - "lat": lat_float, - "lon": lon_float, - "alt": None, - "country": country_name, + "name": _to_text(row.get("name")), + "icao": _to_text(row.get("icao")), + "iata": _to_text(row.get("iata")), + "lat": latitude, + "lon": longitude, + "alt": _to_number(row.get("alt")), + "country": _to_text(row.get("country")), })) + # One line, not one per row: the feed carries every airport. + if unpositioned: + _logger.warning( + "parse_airports_json: %d airport(s) had unusable coordinates and carry no position (e.g. %s).", + len(unpositioned), ", ".join(repr(name) for name in unpositioned[:3]), + ) + + if wanted is not None: + missing = sorted(wanted - matched) + + if missing: + _logger.warning( + "parse_airports_json: no airports found for %s — check the Countries enum.", + ", ".join(missing), + ) + return airports diff --git a/python/FlightRadarAPI/request.py b/python/FlightRadarAPI/request.py index 9f2cff7..991b113 100644 --- a/python/FlightRadarAPI/request.py +++ b/python/FlightRadarAPI/request.py @@ -233,7 +233,23 @@ def get_content(self) -> Union[Dict, bytes]: try: content = decode(content) except Exception as err: - _logger.warning( + # Decided by the body, not the header: undecodable text is genuinely + # broken and must warn, while binary bodies carry no such tell. Nothing + # here may raise, or it would replace `err` with its own failure. + if not isinstance(content, bytes): + transport_decoded = True + elif content_type.startswith(("application/json", "text/")): + try: + content.decode("utf-8") + transport_decoded = True + except UnicodeDecodeError: + transport_decoded = False + else: + corrupt_gzip = content_encoding == "gzip" and content.startswith(b"\x1f\x8b") + transport_decoded = content_encoding in ("gzip", "br") and not corrupt_gzip + + _logger.log( + logging.DEBUG if transport_decoded else logging.WARNING, "APIRequest.get_content: failed to decode Content-Encoding=%r for %s (%s). " "Assuming the transport already decompressed and returning raw bytes.", content_encoding, self.url, err, diff --git a/python/README.md b/python/README.md index a14a185..5c87ffa 100644 --- a/python/README.md +++ b/python/README.md @@ -32,6 +32,7 @@ flights = fr_api.get_flights(...) # Returns a list of Flight objects **Getting airports list:** ```py airports = fr_api.get_airports(...) # Returns a list of Airport objects +all_airports = fr_api.get_airports() # Omit the countries to get every airport ``` **Getting airlines list:** ```py diff --git a/python/tests/fixtures/airports.json b/python/tests/fixtures/airports.json new file mode 100644 index 0000000..924f9db --- /dev/null +++ b/python/tests/fixtures/airports.json @@ -0,0 +1,77 @@ +{ + "version": "1786010717", + "rows": [ + { + "name": "Sao Paulo Guarulhos International Airport", + "iata": "GRU", + "icao": "SBGR", + "lat": -23.429991, + "lon": -46.4674, + "country": "Brazil", + "alt": 2436 + }, + { + "name": "Rio de Janeiro Galeao International Airport", + "iata": "GIG", + "icao": "SBGL", + "lat": -22.805696, + "lon": -43.25523, + "country": "Brazil", + "alt": 22 + }, + { + "name": "Sao Paulo Congonhas Airport", + "iata": "CGH", + "icao": "SBSP", + "lat": -23.627348, + "lon": -46.655987, + "country": "Brazil", + "alt": 2614 + }, + { + "name": "Los Angeles International Airport", + "iata": "LAX", + "icao": "KLAX", + "lat": 33.94252, + "lon": -118.406998, + "country": "United States", + "alt": 125 + }, + { + "name": "New York John F. Kennedy International Airport", + "iata": "JFK", + "icao": "KJFK", + "lat": 40.639751, + "lon": -73.7789, + "country": "United States", + "alt": 14 + }, + { + "name": "A Coruna Airport", + "iata": "LCG", + "icao": "LECO", + "lat": 43.302059, + "lon": -8.37725, + "country": "Spain", + "alt": 326 + }, + { + "name": "Ann Airport", + "iata": "VBA", + "icao": "VYAN", + "lat": 19.770643, + "lon": 94.026405, + "country": "Myanmar (Burma)", + "alt": "43" + }, + { + "name": "Broken Coords Airport", + "iata": "BAD", + "icao": "SBAD", + "lat": "n/a", + "lon": "", + "country": "Brazil", + "alt": 0 + } + ] +} diff --git a/python/tests/fixtures/airports_brazil.html b/python/tests/fixtures/airports_brazil.html deleted file mode 100644 index dc62403..0000000 --- a/python/tests/fixtures/airports_brazil.html +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - -
- São Paulo / Guarulhos (GRU/SBGR) -
- Rio de Janeiro / Galeão (GIG/SBGL) -
- São Paulo / Congonhas (CGH) -
- Bad Coords Airport (BAD/SBBD) -
placeholder
- diff --git a/python/tests/test_api.py b/python/tests/test_api.py index 5c8e0a4..43b605a 100644 --- a/python/tests/test_api.py +++ b/python/tests/test_api.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +import re + import pytest from FlightRadarAPI import Entity, Flight from FlightRadarAPI.errors import CloudflareError, LoginError +from FlightRadarAPI.parsers import country_to_slug from package import Countries, FlightRadar24API, version from util import repeat_test @@ -53,6 +56,27 @@ def test_get_airports(expect=1800, countries=[Countries.BRAZIL, Countries.UNITED assert len(results) >= expect +@repeat_test(**repeat_test_config) +def test_get_airports_without_countries(expect=1800): + results = fr_api.get_airports() + assert len(results) > expect + assert len({airport.country for airport in results}) > 2 + + # Discovered, not hard-coded: FR24 spells some names "Myanmar (Burma)". Sorted + # so the pick does not depend on the order the feed happens to list airports in. + punctuated = sorted(a.country for a in results if re.search(r"[^a-zA-Z ]", a.country)) + tricky = punctuated[0] if punctuated else results[0].country + + assert fr_api.get_country_flag(tricky) is not None, tricky + + # The enum's URL slugs and the feed's display names are two FR24 vocabularies; + # a rename silently empties that country's filter. The reverse direction is not + # asserted: a country FR24 adds is a gap in the enum, not a regression. + absent = {c.value for c in Countries} - {country_to_slug(a.country) for a in results} + + assert not absent, "enum values absent from the feed: %s" % sorted(absent) + + @repeat_test(**repeat_test_config) def test_get_zones(expect=5): results = fr_api.get_zones() diff --git a/python/tests/test_parsers_offline.py b/python/tests/test_parsers_offline.py index 77c3b22..cf675ac 100644 --- a/python/tests/test_parsers_offline.py +++ b/python/tests/test_parsers_offline.py @@ -1,19 +1,36 @@ # -*- coding: utf-8 -*- -"""Offline parser tests against bundled HTML fixtures. +"""Offline parser tests against bundled fixtures. These tests run with no network access and exist specifically so that PRs can be gated on the parser logic without depending on FR24 being reachable -or its HTML layout being stable. When FR24 changes the page structure, -update the fixtures (re-saving a real page is fine) — the assertions here +or its payloads being stable. When FR24 changes a page or feed, +update the fixtures (re-saving a real response is fine) — the assertions here guard the parser's invariants, not byte-for-byte equality with production. """ +import json +import math import os -from FlightRadarAPI.parsers import parse_airlines_html, parse_airports_html +import pytest + +from FlightRadarAPI.core import Countries +from FlightRadarAPI.parsers import country_to_slug, parse_airlines_html, parse_airports_json FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") +# Keep in step with the same list in nodejs/tests/testParsersOffline.js. +NUMERIC_CASES = [ + ("whitespace", " ", None), + ("an array holding a number", [43], None), + ("a coordinate with a hemisphere suffix", "43.30 N", None), + ("an exponent overflowing a double", "1e999", None), + ("400 plain digits", "1" * 400, None), + ("a whole number as a string", "2436", 2436), + ("a negative decimal", "-23.4", -23.4), + ("a genuine zero", 0, 0), +] + def _load(name: str) -> bytes: with open(os.path.join(FIXTURES, name), "rb") as f: @@ -63,42 +80,175 @@ def test_parse_airlines_html_empty_input_returns_empty_list(): assert parse_airlines_html(b"") == [] assert parse_airlines_html(b"

no tbody here

") == [] +# --- parse_airports_json --- -# --- parse_airports_html --- -def test_parse_airports_html_extracts_basic_fields(): - airports = parse_airports_html(_load("airports_brazil.html"), "/data/airports/brazil") +def test_parse_airports_json_extracts_basic_fields(): + airports = parse_airports_json(_load("airports.json")) by_iata = {a.iata: a for a in airports} assert "GRU" in by_iata and "GIG" in by_iata gru = by_iata["GRU"] assert gru.icao == "SBGR" assert gru.country == "Brazil" - assert abs(gru.latitude - (-23.4356)) < 1e-6 - assert abs(gru.longitude - (-46.4731)) < 1e-6 + assert abs(gru.latitude - (-23.429991)) < 1e-6 + assert abs(gru.longitude - (-46.4674)) < 1e-6 + assert gru.altitude == 2436 + +def test_parse_airports_json_keeps_every_country_without_filter(): + countries = {a.country for a in parse_airports_json(_load("airports.json"))} + assert {"Brazil", "United States", "Spain"} <= countries -def test_parse_airports_html_handles_iata_only_small_tag(): - airports = parse_airports_html(_load("airports_brazil.html"), "/data/airports/brazil") - cgh = next(a for a in airports if a.iata == "CGH") - assert cgh.icao == "" + +def test_parse_airports_json_filters_by_country_slug(): + airports = parse_airports_json(_load("airports.json"), ["united-states"]) + assert airports + assert all(a.country == "United States" for a in airports) -def test_parse_airports_html_invalid_coordinates_become_none(): +def test_parse_airports_json_accepts_several_countries(): + airports = parse_airports_json(_load("airports.json"), ["brazil", "spain"]) + assert {a.country for a in airports} == {"Brazil", "Spain"} + + +def test_parse_airports_json_invalid_coordinates_become_none(): """Regression: invalid coords used to be silently coerced to (0.0, 0.0) placing the airport in the Gulf of Guinea. They must be None now.""" - airports = parse_airports_html(_load("airports_brazil.html"), "/data/airports/brazil") + airports = parse_airports_json(_load("airports.json")) bad = next(a for a in airports if a.iata == "BAD") assert bad.latitude is None assert bad.longitude is None -def test_parse_airports_html_empty_input_returns_empty_list(): - assert parse_airports_html(b"", "/data/airports/brazil") == [] +def test_parse_airports_json_rejects_numeric_looking_junk(): + payload = json.dumps({"rows": [{ + "name": "Junk Airport", "iata": "JNK", "icao": "JJNK", + "lat": "43.30 N", "lon": -8.37725, "country": "Spain", "alt": "-1", + }]}) + airport = parse_airports_json(payload)[0] + + # One bad coordinate drops both; altitude is independent and survives. + assert airport.latitude is None + assert airport.longitude is None + assert airport.altitude == -1 + assert isinstance(airport.altitude, int) + + +@pytest.mark.parametrize("value,expected", [ + ("٤٣", None), # Arabic-Indic digits, which \d matched before re.ASCII + ("43", None), # U+FEFF, which String.trim() drops and str.strip() keeps + ("43", None), # U+001C, which str.strip() drops and String.trim() keeps + (" 43 ", 43), # plain spaces are still trimmed +]) +def test_parse_airports_json_agrees_with_node_on_unicode(value, expected): + payload = json.dumps({"rows": [{ + "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", + "lat": value, "lon": value, "alt": value, + }]}) + + assert parse_airports_json(payload)[0].altitude == expected + + +@pytest.mark.parametrize( + "value,expected", + [(case[1], case[2]) for case in NUMERIC_CASES], + ids=[case[0] for case in NUMERIC_CASES], +) +def test_parse_airports_json_numeric_coercion(value, expected): + payload = json.dumps({"rows": [{ + "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", + "lat": value, "lon": value, "alt": value, + }]}) + airport = parse_airports_json(payload)[0] + + assert airport.latitude == expected + assert airport.longitude == expected + assert airport.altitude == expected + + +@pytest.mark.parametrize("literal", ["1" * 400, "9007199254740993", "-" + "1" * 400]) +def test_parse_airports_json_survives_raw_numbers_no_double_can_hold(literal): + """Written as text: json.loads turns these into unbounded ints.""" + payload = ( + '{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",' + f'"lat":{literal},"lon":1,"alt":2}}]}}' + ) + latitude = parse_airports_json(payload)[0].latitude + + assert latitude is None or math.isfinite(latitude) -def test_parse_airports_html_derives_country_from_href(): - airports = parse_airports_html( - _load("airports_brazil.html"), "/data/airports/united-states", +@pytest.mark.parametrize("literal", ["null", "0", "false", "true", "[]", "{}", "123"]) +def test_parse_airports_json_keeps_text_fields_as_strings(literal): + """Anything else breaks get_country_flag(airport.country).""" + payload = ( + '{"rows":[{"name":%s,"iata":%s,"icao":%s,"country":%s,"lat":1,"lon":2,"alt":3}]}' + % (literal, literal, literal, literal) ) - assert all(a.country == "United States" for a in airports) + airport = parse_airports_json(payload)[0] + + assert [airport.name, airport.iata, airport.icao, airport.country] == ["", "", "", ""] + + +def test_parse_airports_json_country_spelling_and_slug_round_trip(): + ann = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "VBA") + + assert ann.country == "Myanmar (Burma)" + assert country_to_slug(ann.country) == "myanmar-burma" + assert ann.altitude == 43 + + +def test_parse_airports_json_filter_accepts_either_country_spelling(): + for slug in ["myanmar-burma", "Myanmar (Burma)"]: + assert len(parse_airports_json(_load("airports.json"), [slug])) == 1 + + +def test_parse_airports_json_keeps_altitude_as_int(): + """Both ports must agree on altitude: 2436, never 2436.0.""" + gru = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "GRU") + assert isinstance(gru.altitude, int) + + +def test_parse_airports_json_skips_rows_that_are_not_objects(): + payload = ( + '{"rows":[[1,2,3],"x",7,null,' + '{"name":"Real","iata":"RRR","icao":"RRRR","country":"Spain","lat":1,"lon":2,"alt":3}]}' + ) + + assert [a.iata for a in parse_airports_json(payload)] == ["RRR"] + + +def test_parse_airports_json_unknown_country_returns_empty_list(): + assert parse_airports_json(_load("airports.json"), ["atlantis"]) == [] + + +def test_parse_airports_json_invalid_payload_returns_empty_list(): + assert parse_airports_json(b"") == [] + assert parse_airports_json(b"{}") == [] + assert parse_airports_json(b"not json") == [] + + +# --- country_to_slug --- + +def test_country_to_slug_matches_countries_enum_spelling(): + assert country_to_slug("United States") == "united-states" + assert country_to_slug("Democratic Republic Of The Congo") == "democratic-republic-of-the-congo" + assert country_to_slug("Curacao") == "curacao" + assert country_to_slug("Curaçao") == "curacao" + assert country_to_slug(None) == "" + + +def test_country_to_slug_takes_a_countries_member(): + """get_airports accepts them, so get_country_flag(airport.country) style calls + must not slugify "Countries.BRAZIL".""" + assert country_to_slug(Countries.BRAZIL) == "brazil" + assert country_to_slug(Countries.MYANMAR_BURMA) == "myanmar-burma" + + +def test_country_to_slug_strips_parentheses_for_flag_urls(): + """These 404'd while the slug was a plain space-to-hyphen replacement.""" + assert country_to_slug("Myanmar (Burma)") == "myanmar-burma" + assert country_to_slug("Cocos (Keeling) Islands") == "cocos-keeling-islands" + assert country_to_slug("Falkland Islands (Malvinas)") == "falkland-islands-malvinas" + assert country_to_slug("Timor-Leste (East Timor)") == "timor-leste-east-timor"