Calorie Tape: how the app's Open Food Facts data is made ========================================================= Calorie Tape bundles a modified copy of Open Food Facts product data, available under the Open Database License (ODbL) 1.0: https://opendatacommons.org/licenses/odbl/1-0/ This file is the method that makes it, as ODbL section 4.6 asks. Source: the Open Food Facts export openfoodfacts-products.jsonl.gz, downloaded 2026-09-23, SHA-256 7f09bece18417fb14b90c32ae3e4dabd159a633dca9c51bbb7b1feda32061b78 Steps: - Keep products sold in the United States, Canada, the United Kingdom, Australia or New Zealand. - Normalize barcodes to GTIN-14 and drop codes that aren't barcodes; for a barcode listed twice, keep the most recently edited product. - Drop barcodes that USDA Branded Foods has. - Keep reported values per 100 g or 100 ml only: no ingredient-list estimates, no vitamin A, and no single value over 100 g per 100. - Require a name, energy, protein, fat and carbohydrates; drop products over 950 kcal, with any of those three over 100 g, or summing past 110 g per 100. - Drop products with less energy than their fat alone supplies: under 6.75 kcal per gram of fat, less 10 kcal. - Drop products whose two package columns, as sold, disagree on energy: the energy per 100 g (or 100 ml), scaled to the serving, and the energy per serving differ by more than a quarter and by more than 5 kcal. Only this check reads the per-serving energy. - Keep the label serving when it's in the food's own unit (g or ml), the English name when there is one, and the first listed brand. The program that runs these steps follows: off.ts, then the helpers it uses from transform.ts. // ===== off.ts ===== // Open Food Facts: packaged products sold in the app's storefronts that USDA Branded doesn't have (spec: Open Food Facts). import { mkdir, rename, rm } from "node:fs/promises"; import { dirname } from "node:path"; import { Basis, countDrop, deriveDensity, joinedSpellings, normalizeGTIN, normalizeNutrients, rejectionReason, Source, type DropCounts, type FoodRecord, type NutrientRules, } from "./transform"; /** OFF's `countries_tags` for the App Store storefronts the app is sold in. */ export const storefronts = ["en:united-states", "en:canada", "en:united-kingdom", "en:australia", "en:new-zealand"]; /** * OFF nutrient → USDA nutrient id, and the factor from OFF's per-100 unit (grams; kcal for energy) to the unit in * nutrients.json. Vitamin A is left out: OFF's entries mix IU and µg through conversions that aren't exact. */ const nutrientMap: [offName: string, id: number, factor: number][] = [ ["energy-kcal", 1008, 1], ["proteins", 1003, 1], ["fat", 1004, 1], ["carbohydrates", 1005, 1], ["sugars", 2000, 1], ["added-sugars", 1235, 1], ["fiber", 1079, 1], ["saturated-fat", 1258, 1], ["trans-fat", 1257, 1], ["monounsaturated-fat", 1292, 1], ["polyunsaturated-fat", 1293, 1], ["cholesterol", 1253, 1_000], ["sodium", 1093, 1_000], ["potassium", 1092, 1_000], ["calcium", 1087, 1_000], ["iron", 1089, 1_000], ["magnesium", 1090, 1_000], ["phosphorus", 1091, 1_000], ["zinc", 1095, 1_000], ["vitamin-c", 1162, 1_000], ["vitamin-d", 1114, 1_000_000], ["caffeine", 1057, 1_000], ]; const kilojoulesPerKilocalorie = 4.184; /** Grams per unit of a newer-schema nutrient value, so both schemas stage in OFF's per-100 gram convention. */ const gramsPerUnit: Record = { g: 1, mg: 1e-3, "µg": 1e-6, mcg: 1e-6, kcal: 1, kj: 1 }; /** Bumped whenever `stagedProduct` changes, so a staging file written by older code is rebuilt. */ const stagingFormat = 3; /** The fields the catalog reads from an OFF product, as staged. Everything else in the dump is dropped. */ export interface OffProduct { code: string; product_name_en?: string; product_name?: string; brands?: string; /** Per-100 values, keyed like the dump's `nutriments` ("sodium_100g"). */ nutriments: Record; /** Nutrients OFF estimated from the ingredient list; unknown here, never values. */ estimated: string[]; /** The nutrition table's own basis, "100g" or "100ml", when OFF records one. */ per?: string; serving_size?: string; serving_quantity?: number; serving_quantity_unit?: string; product_quantity_unit?: string; last_modified_t?: number; /** As-sold energy from the package's per-100 and per-serving columns, when OFF records both. */ packageEnergy?: PackageEnergy; } /** Energy (kcal) a package prints per 100 g or ml, and per serving of `servingQuantity` in the same unit. */ export interface PackageEnergy { per100: number; perServing: number; servingQuantity: number; } /** * Stages the storefront products from the dump (once per `stagingKey`), then returns one record per barcode that * `usdaGTINs` doesn't already have. `resolveDump` is called only when the staging file is missing or stale. */ export async function readOff( resolveDump: () => Promise, stagingPath: string, stagingKey: string, usdaGTINs: Set, rules: NutrientRules, drops: DropCounts, ): Promise { if (!(await isStaged(stagingPath, stagingKey))) await stage(await resolveDump(), stagingPath, stagingKey); const newest = new Map(); let isHeader = true; for await (const line of lines(stagingPath)) { if (isHeader) { isHeader = false; continue; } const product = JSON.parse(line) as OffProduct; const gtin = normalizeGTIN(product.code); if (!gtin) { countDrop(drops, "off.invalidGTIN"); continue; } if (usdaGTINs.has(gtin)) { countDrop(drops, "off.inUSDA"); continue; } const current = newest.get(gtin); if (current) countDrop(drops, "off.olderDuplicate"); if (!current || (product.last_modified_t ?? 0) > (current.last_modified_t ?? 0)) newest.set(gtin, product); } const records: FoodRecord[] = []; for (const [gtin, product] of newest) { const outcome = toOffRecord(gtin, product, rules); if ("record" in outcome) records.push(outcome.record); else countDrop(drops, `off.${outcome.rejected}`); } return records; } /** Whether the dump's product is sold in one of the app's storefronts. */ export function isSoldInStorefront(product: { countries_tags?: string[] }): boolean { return (product.countries_tags ?? []).some(tag => storefronts.includes(tag)); } /** * Reduces a dump product to the fields the catalog reads, recording which nutrients OFF estimated. Older records keep * per-100 values in `nutriments`; newer ones only in `nutrition.aggregated_set`, each with its own unit. */ export function stagedProduct(raw: any): OffProduct { const nutriments: Record = {}; for (const [name] of nutrientMap) copyNumber(raw.nutriments, `${name}_100g`, nutriments); copyNumber(raw.nutriments, "energy_100g", nutriments); const aggregated = raw.nutrition?.aggregated_set; const entries = Object.entries(aggregated?.nutrients ?? {}) as [string, any][]; const estimated = entries.filter(([, entry]) => entry?.source === "estimate").map(([name]) => name); if (aggregated?.per === "100g" || aggregated?.per === "100ml") { for (const [name, entry] of entries) { const grams = gramsPerUnit[String(entry?.unit ?? "").toLowerCase()]; const value = number(entry?.value); if (entry?.source === "estimate" || grams === undefined || value === undefined) continue; nutriments[`${name}_100g`] ??= value * grams; } } return { code: String(raw.code ?? ""), product_name_en: text(raw.product_name_en), product_name: text(raw.product_name), brands: text(raw.brands), nutriments, estimated, per: text(aggregated?.per), serving_size: text(raw.serving_size), serving_quantity: number(raw.serving_quantity), serving_quantity_unit: text(raw.serving_quantity_unit), product_quantity_unit: text(raw.product_quantity_unit), last_modified_t: number(raw.last_modified_t), packageEnergy: packageEnergy(raw.nutrition?.input_sets), }; } /** Energy from the as-sold columns typed from the package, when both a per-100 and a per-serving one share a unit. */ function packageEnergy(inputSets: unknown): PackageEnergy | undefined { const columns = (Array.isArray(inputSets) ? inputSets : []) .filter(set => set?.source === "packaging" && set?.preparation === "as_sold"); const kcal = (set: any) => { const entry = set?.nutrients?.["energy-kcal"]; return String(entry?.unit ?? "").toLowerCase() === "kcal" ? number(entry.value) : undefined; }; const per100 = columns.find(set => set.per === "100g" || set.per === "100ml"); const serving = columns.find(set => set.per === "serving" && `100${String(set.per_unit ?? "").toLowerCase()}` === per100?.per); const per100Kcal = kcal(per100); const servingKcal = kcal(serving); const servingQuantity = number(serving?.per_quantity); if (per100Kcal === undefined || servingKcal === undefined || servingQuantity === undefined || !(servingQuantity > 0)) return undefined; return { per100: per100Kcal, perServing: servingKcal, servingQuantity }; } /** * Whether a package's per-100 and per-serving energies can't both be true: they differ by more than a quarter and * more than 5 kcal, which label rounding doesn't reach. One column holds the other's values. */ export function packageColumnsDisagree({ per100, perServing, servingQuantity }: PackageEnergy): boolean { const scaled = per100 * servingQuantity / 100; const [low, high] = scaled < perServing ? [scaled, perServing] : [perServing, scaled]; return high - low > 5 && high > 1.25 * low; } export type OffOutcome = { record: FoodRecord } | { rejected: string }; export function toOffRecord(gtin: string, product: OffProduct, rules: NutrientRules): OffOutcome { const name = collapse(product.product_name_en) || collapse(product.product_name); if (!name) return { rejected: "emptyName" }; const nutrients = normalizeNutrients(offNutrients(product), rules); const rejection = rejectionReason(nutrients); if (rejection) return { rejected: rejection }; if (product.packageEnergy && packageColumnsDisagree(product.packageEnergy)) return { rejected: "columnsDisagree" }; const basis = offBasis(product); const brands = (product.brands ?? "").split(",").map(collapse).filter(Boolean); const serving = offServing(product, basis); return { record: { key: `gtin:${gtin}`, source: Source.openFoodFacts, name, brand: brands[0], keywords: [...brands.slice(1), joinedSpellings(brands)].filter(Boolean).join(" ") || undefined, basis, // Only gram-based foods gain volume units from a density; ml-based foods already measure volume. density: basis === Basis.grams && serving ? deriveDensity([{ label: serving.label, grams: serving.amount }]) : undefined, nutrients, portions: serving ? [{ key: "serving", label: serving.label, amount: serving.amount, seq: 1 }] : [], }, }; } /** * Reported per-100 values in the curated units. An estimated nutrient is unknown, and so is a value over 100 g per * 100, which no food can have. Energy given only in kJ converts exactly. */ export function offNutrients(product: OffProduct): Map { const estimated = new Set(product.estimated); const values = new Map(); for (const [name, id, factor] of nutrientMap) { const amount = product.nutriments[`${name}_100g`]; if (amount === undefined || estimated.has(name)) continue; if (id !== 1008 && amount > 100) continue; values.set(id, amount * factor); } const kilojoules = product.nutriments["energy_100g"]; if (!values.has(1008) && kilojoules !== undefined && !estimated.has("energy") && !estimated.has("energy-kcal")) { values.set(1008, kilojoules / kilojoulesPerKilocalorie); } return values; } /** Per 100 ml for products OFF measures in ml: the table's own basis, else the serving's unit, else the package's. */ export function offBasis(product: OffProduct): Basis { if (product.per === "100ml") return Basis.millilitres; if (product.per === "100g") return Basis.grams; const unit = (product.serving_quantity_unit ?? product.product_quantity_unit ?? "").toLowerCase(); return unit === "ml" ? Basis.millilitres : Basis.grams; } /** The label serving, when its quantity is in the food's basis unit. Its label keeps the household part only. */ export function offServing(product: OffProduct, basis: Basis): { label: string; amount: number } | undefined { const amount = product.serving_quantity; const unit = basis === Basis.grams ? "g" : "ml"; if (amount === undefined || !(amount > 0) || product.serving_quantity_unit?.toLowerCase() !== unit) return undefined; return { label: householdServing(product.serving_size ?? "") ?? `${amount} ${unit}`, amount }; } /** * The household part of a serving, without the metric amount the unit menu already shows: * "44 g (1 cup)" → "1 cup", "1 cup (240 ml)" → "1 cup". A metric amount alone has none. */ export function householdServing(serving: string): string | undefined { const metric = String.raw`[\d.,]+\s*(?:g|gr|grams?|ml|mls?)\.?`; const text = collapse(serving); const inner = new RegExp(String.raw`^${metric}\s*\((.+)\)$`, "i").exec(text); if (inner) return collapse(inner[1]) || undefined; const outer = new RegExp(String.raw`^(.+?)\s*\(\s*${metric}\s*\)$`, "i").exec(text); if (outer) return collapse(outer[1]) || undefined; if (new RegExp(String.raw`^${metric}$`, "i").test(text)) return undefined; return text || undefined; } async function isStaged(stagingPath: string, stagingKey: string): Promise { if (!(await Bun.file(stagingPath).exists())) return false; for await (const line of lines(stagingPath)) { const header = JSON.parse(line); return header.stagingKey === stagingKey && header.format === stagingFormat; } return false; } /** Streams the dump once, keeping storefront products' catalog fields. The first line records the key and format. */ async function stage(dumpPath: string, stagingPath: string, stagingKey: string) { console.log("Staging Open Food Facts products (streams the whole dump)…"); await mkdir(dirname(stagingPath), { recursive: true }); const partial = `${stagingPath}.partial`; const writer = Bun.file(partial).writer(); writer.write(JSON.stringify({ stagingKey, format: stagingFormat }) + "\n"); const tags = storefronts.map(tag => `"${tag}"`); for await (const line of lines(dumpPath)) { // Most of the dump is sold elsewhere; skip parsing it. if (!tags.some(tag => line.includes(tag))) continue; const raw = JSON.parse(line); if (isSoldInStorefront(raw)) writer.write(JSON.stringify(stagedProduct(raw)) + "\n"); } await writer.end(); await rm(stagingPath, { force: true }); await rename(partial, stagingPath); } /** The file's lines, gunzipped by a separate process when it ends in `.gz`. */ async function* lines(path: string): AsyncGenerator { const gunzip = path.endsWith(".gz") ? Bun.spawn(["gzip", "-dc", path], { stdout: "pipe", stderr: "inherit" }) : undefined; const stream = gunzip ? gunzip.stdout : Bun.file(path).stream(); const decoder = new TextDecoder(); let buffered = ""; try { for await (const chunk of stream) { buffered += decoder.decode(chunk, { stream: true }); let newline: number; while ((newline = buffered.indexOf("\n")) >= 0) { const line = buffered.slice(0, newline); buffered = buffered.slice(newline + 1); if (line) yield line; } } buffered += decoder.decode(); if (buffered) yield buffered; } finally { gunzip?.kill(); } if (gunzip && (await gunzip.exited) !== 0 && gunzip.signalCode === null) throw new Error(`gzip failed on ${path}`); } function copyNumber(from: Record | undefined, key: string, into: Record) { const value = from?.[key]; const parsed = typeof value === "string" ? Number(value) : value; if (typeof parsed === "number" && Number.isFinite(parsed)) into[key] = parsed; } function text(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value : undefined; } function number(value: unknown): number | undefined { const parsed = typeof value === "string" ? Number(value) : value; return typeof parsed === "number" && Number.isFinite(parsed) ? parsed : undefined; } function collapse(text: string | undefined): string { return (text ?? "").replace(/\s+/g, " ").trim(); } // ===== transform.ts ===== // Pure data transforms for the catalog build. Everything here is deterministic and unit-tested. /** Matches the app's `FoodSource`, where 3 is the user's custom foods, which never ship in the catalog. */ export enum Source { fndds = 1, branded = 2, openFoodFacts = 4 } export enum Basis { grams = 0, millilitres = 1 } export interface PortionRecord { key: string; label: string; /** In the food's basis unit: grams, or millilitres for ml-basis foods. */ amount: number; seq: number; } export interface FoodRecord { key: string; source: Source; name: string; brand?: string; /** Extra search-only text, e.g. joined brand spellings. Never displayed. */ keywords?: string; basis: Basis; density?: number; category?: string; /** Per 100 basis units. An absent id means unknown, never zero. */ nutrients: Map; portions: PortionRecord[]; } /** Counts of excluded source rows by reason, reported after every build. */ export type DropCounts = Record; export function countDrop(drops: DropCounts, reason: string) { drops[reason] = (drops[reason] ?? 0) + 1; } export const Nutrient = { energy: 1008, protein: 1003, fat: 1004, carbohydrate: 1005, vitaminDMicrograms: 1114, vitaminDInternationalUnits: 1110, } as const; /** Barcodes are keyed as GTIN-14 so UPC-A, EAN-13, and EAN-8 spellings of one product collide. */ export function normalizeGTIN(raw: string): string | undefined { const digits = raw.trim(); if (!/^\d{1,14}$/.test(digits)) return undefined; return digits.padStart(14, "0"); } /** * Branded descriptions often repeat the flavor: "CRACKERS, FOUR CHEESE, FOUR CHEESE" or * "HUMMUS, ROASTED RED PEPPER" where the prefix already says "ROASTED RED PEPPER HUMMUS". * Drop trailing comma segments whose words already appear in the rest of the name. */ export function cleanName(raw: string): string { let name = raw.replace(/\s+/g, " ").trim(); for (;;) { const comma = name.lastIndexOf(","); if (comma <= 0) return name; const head = name.slice(0, comma).trim(); const tail = name.slice(comma + 1).trim(); if (!tail || !containsWords(head, tail)) return name; name = head; } } /** "FAGE" + "BESTSELF" → "FAGE BESTSELF"; a sub-brand the brand already names is not repeated. */ export function composeBrand(brand: string, subbrand: string): string | undefined { if (!subbrand) return brand || undefined; if (!brand) return subbrand; return containsWords(brand, subbrand) ? brand : `${brand} ${subbrand}`; } /** * Joined spellings of multi-word brands ("Best Self" → "BestSelf", "Trader Joe's" → "TraderJoes"), indexed so a * query typed as one word still matches. The app handles the opposite direction by also joining adjacent query terms. */ export function joinedSpellings(texts: (string | undefined)[]): string | undefined { const joined = new Set(); for (const text of texts) { const words = (text ?? "").split(/[^\p{L}\p{N}]+/u).filter(Boolean); if (words.length > 1) joined.add(words.join("")); } return joined.size ? [...joined].join(" ") : undefined; } function containsWords(text: string, phrase: string): boolean { const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return new RegExp(`(^|[^\\p{L}\\p{N}])${escaped}($|[^\\p{L}\\p{N}])`, "iu").test(text); } // US customary volumes, in millilitres. const millilitres: [RegExp, number][] = [ [/^(fl\.? ?oz|fluid ounces?)\b/i, 29.5735], [/^(cups?|c)\b/i, 236.588], [/^(tablespoons?|tbsps?|tbs)\b/i, 14.7868], [/^(teaspoons?|tsps?)\b/i, 4.92892], [/^(ml|mls|millilit(er|re)s?)\b/i, 1], [/^(l|lit(er|re)s?)\b/i, 1000], [/^(pints?|pt)\b/i, 473.176], [/^(quarts?|qt)\b/i, 946.353], [/^(gallons?|gal)\b/i, 3785.41], ]; /** Parses the volume a portion label describes, e.g. "1 cup, chopped" or "0.25 CUP" or "1 1/2 tbsp". */ export function parseVolume(label: string): number | undefined { const match = /^\s*(\d+\s+\d+\/\d+|\d+\/\d+|\d*\.?\d+)\s*(.*)$/.exec(label); if (!match) return undefined; const quantity = parseQuantity(match[1]); if (!quantity) return undefined; const rest = match[2]; for (const [pattern, ml] of millilitres) { if (pattern.test(rest)) return quantity * ml; } return undefined; } function parseQuantity(text: string): number | undefined { const parts = text.trim().split(/\s+/); let total = 0; for (const part of parts) { const [numerator, denominator] = part.split("/"); const value = denominator === undefined ? Number(numerator) : Number(numerator) / Number(denominator); if (!Number.isFinite(value)) return undefined; total += value; } return total > 0 ? total : undefined; } /** Density in g/ml from portions whose label is a volume. Median, to shrug off one odd portion. */ export function deriveDensity(portions: { label: string; grams: number }[]): number | undefined { const densities = portions.flatMap(({ label, grams }) => { const ml = parseVolume(label); return ml && grams > 0 ? [grams / ml] : []; }).sort((a, b) => a - b); if (!densities.length) return undefined; const middle = Math.floor(densities.length / 2); const median = densities.length % 2 ? densities[middle] : (densities[middle - 1] + densities[middle]) / 2; // Outside this range the label or weight is wrong, not the food. return median >= 0.1 && median <= 3 ? round(median, 4) : undefined; } /** From nutrients.json: ids that duplicate another id (aliases) and ids the app never stores (excluded). */ export interface NutrientRules { aliases: Map; excluded: Set; } /** * Applies aliases and exclusions, converts IU vitamin D to µg (exact: 1 µg = 40 IU; µg wins when both exist), * and drops values that are not finite non-negative numbers. Missing stays missing. */ export function normalizeNutrients(values: Map, rules: NutrientRules): Map { const result = new Map(); for (const [rawId, amount] of values) { if (!Number.isFinite(amount) || amount < 0 || rules.excluded.has(rawId)) continue; const id = rules.aliases.get(rawId) ?? rawId; if (!result.has(id) || id === rawId) result.set(id, amount); } const vitaminDIU = result.get(Nutrient.vitaminDInternationalUnits); result.delete(Nutrient.vitaminDInternationalUnits); if (vitaminDIU !== undefined && !result.has(Nutrient.vitaminDMicrograms)) { result.set(Nutrient.vitaminDMicrograms, vitaminDIU / 40); } return result; } export type Rejection = "missingMacros" | "implausible" | "energyBelowFat"; /** Values are per 100 g or 100 ml. Returns why a food should be excluded, if it should. */ export function rejectionReason(values: Map): Rejection | undefined { const energy = values.get(Nutrient.energy); const macros = [Nutrient.protein, Nutrient.fat, Nutrient.carbohydrate].map(id => values.get(id)); if (energy === undefined || macros.some(value => value === undefined)) return "missingMacros"; const [protein, fat, carbohydrate] = macros as number[]; // Pure fats reach ~902 kcal/100 g, and label rounding pushes small-serving oils toward 940; above 950 is bad data. if (energy > 950 || protein > 100 || fat > 100 || carbohydrate > 100) return "implausible"; if (protein + fat + carbohydrate > 110) return "implausible"; // Fat alone supplies at least 8.37 kcal/g. Below three quarters of 9, less 10 kcal for label rounding, the energy is wrong. if (energy < 6.75 * fat - 10) return "energyBelowFat"; return undefined; } /** Packs nutrients as little-endian (UInt16 id, Float32 amount) pairs sorted by id. */ export function packNutrients(values: Map): Uint8Array { const ids = [...values.keys()].sort((a, b) => a - b); const buffer = new DataView(new ArrayBuffer(ids.length * 6)); ids.forEach((id, index) => { if (!Number.isInteger(id) || id < 0 || id > 0xffff) throw new Error(`Nutrient id out of range: ${id}`); buffer.setUint16(index * 6, id, true); buffer.setFloat32(index * 6 + 2, values.get(id)!, true); }); return new Uint8Array(buffer.buffer); } export function unpackNutrients(bytes: Uint8Array): Map { if (bytes.byteLength % 6 !== 0) throw new Error("Nutrient blob length is not a multiple of 6."); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const values = new Map(); for (let offset = 0; offset < bytes.byteLength; offset += 6) { values.set(view.getUint16(offset, true), view.getFloat32(offset + 2, true)); } return values; } function round(value: number, places: number): number { const factor = 10 ** places; return Math.round(value * factor) / factor; }