Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ bundle:
rm -rf $(TMP_WORKSPACE)/node_modules/core-js/client
rm -rf $(TMP_WORKSPACE)/node_modules/core-js/stage
rm -rf $(TMP_WORKSPACE)/node_modules/nan
find $(TMP_WORKSPACE)/node_modules/adblock-rs/js/target/release/ -type f -not -name libadblock_rs.so -delete
find $(TMP_WORKSPACE)/node_modules/adblock-rs/target/release/ -mindepth 1 -not -name index.js -not -name index.node -exec rm -rf {} +
find $(TMP_WORKSPACE)/node_modules -type f -name "*.md" -delete
find $(TMP_WORKSPACE)/node_modules -type d -name "test" | xargs rm -rf
cd $(TMP_WORKSPACE)/ && zip -r $(FUNCTION_NAME).zip *
Expand Down
26 changes: 24 additions & 2 deletions brave/adblock.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ const adblockRsLib = require('adblock-rs')

const braveDebugLib = require('./debug')

// Non-network request URL schemes that adblock-rs can't parse a hostname from
// (Request::new returns HostnameParseError). These are browser-internal or inline
// resources, not real network requests, and show up routinely in crawl data. Drop
// them so they don't abort the batch; anything else that fails to parse is
// unexpected and should surface (with its value) so we can narrow it down.
const NON_NETWORK_SCHEME_RE = /^(?:data|blob|about|javascript):/i

const serializeRules = rules => {
braveDebugLib.verbose(`Serializing ${rules.length} rules`)
const filterSet = new adblockRsLib.FilterSet(true)
Expand All @@ -17,7 +24,7 @@ const serializeRules = rules => {
optimize: false
}
const adBlockClient = new adblockRsLib.Engine(filterSet, adBlockArgs)
const adBlockDat = adBlockClient.serializeRaw()
const adBlockDat = adBlockClient.serialize()
const adBlockDatBuffer = Buffer.from(adBlockDat)
braveDebugLib.verbose(`Successfully serialized rules into buffer of length ${adBlockDatBuffer.byteLength}`)
return adBlockDatBuffer
Expand All @@ -40,7 +47,22 @@ const applyBlockingRules = (adblockClient, requests) => {
const requestType = aReport[4]
const requestUrl = aReport[5]

const matchResult = adblockClient.check(requestUrl, frameUrl, requestType, true)
// Drop known non-network schemes up front: adblock-rs would throw
// "hostname parsing failed" on these, and they aren't real requests to block.
if (NON_NETWORK_SCHEME_RE.test(requestUrl)) {
braveDebugLib.verbose(`Dropping non-network request ${requestUrl} in frame ${frameUrl}`)
continue
}

let matchResult
try {
matchResult = adblockClient.check(requestUrl, frameUrl, requestType, true)
} catch (err) {
// Not a known non-network scheme, so this parse failure is unexpected.
// Re-throw with the offending values attached so Sentry shows what triggered
// it (the bare adblock-rs message doesn't include the URL).
throw new Error(`adblock check failed for requestUrl=${JSON.stringify(requestUrl)} frameUrl=${JSON.stringify(frameUrl)} requestType=${JSON.stringify(requestType)}: ${err.message}`, { cause: err })
}
if (matchResult.matched === false) {
if (matchResult.exception) {
braveDebugLib.verbose(`Would block ${requestUrl} in frame ${frameUrl} of type ${requestType} with rule ${matchResult.filter} but excepted by ${matchResult.exception}`)
Expand Down
2 changes: 1 addition & 1 deletion brave/lambda_actions/assemble.js
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ const convertRules = (rules, format) => {
const iosFilterSet = new FilterSet(true)
iosFilterSet.addFilters(filtersUsed)
const engine = new Engine(iosFilterSet, { optimize: false })
const iosDat = engine.serializeRaw()
const iosDat = engine.serialize()
const datBuffer = Buffer.from(iosDat)

if (datBuffer.byteLength === 0) {
Expand Down
2 changes: 1 addition & 1 deletion brave/lambda_actions/crawl-dispatch.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ const braveValidationLib = require('../validation')
* the crawl.
*/
const validateArgs = async inputArgs => {
const { v4: genUuid4 } = require('uuid')
const { randomUUID: genUuid4 } = require('node:crypto')
const isString = braveValidationLib.ofTypeAndTruthy.bind(undefined, 'string')
const isAllString = braveValidationLib.allOfTypeAndTruthy.bind(undefined, 'string')

Expand Down
20 changes: 11 additions & 9 deletions brave/lambda_actions/crawl.js
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ const _crawlPage = async (page, args) => {
braveDebugLib.verbose(`Requesting url: ${args.url}`)
await page.goto(args.url)
} catch (e) {
if ((e instanceof puppeteerLib.errors.TimeoutError) === false) {
if ((e instanceof puppeteerLib.TimeoutError) === false) {
braveDebugLib.log(`Error doing top level fetch: ${e.toString()}`)
return
}
Expand All @@ -245,10 +245,10 @@ const _crawlPage = async (page, args) => {
if (timeElapsed < waitTime) {
const additionalWaitTime = waitTime - timeElapsed
braveDebugLib.verbose(`Waiting an extra: ${additionalWaitTime}ms`)
await page.waitForTimeout(additionalWaitTime)
await new Promise(resolve => setTimeout(resolve, additionalWaitTime))
}

page.removeListener('requestfinished', callbackHandler)
page.off('requestfinished', callbackHandler)
braveDebugLib.verbose(`Captured ${report.length} requests.`)

// Check to see if we should go "deeper"
Expand Down Expand Up @@ -317,14 +317,16 @@ const start = async args => {
}

braveDebugLib.verbose('Wrapping up and closing puppeteer.')
// browser.close() can hang when Chromium wedges, so cap how long we block on it.
// The timeout only stops us waiting; it does not cancel close(), which keeps
// running in the background (along with puppeteer's own temp userDataDir cleanup),
// so if the environment is reused the shutdown can still finish.
try {
browser.close()
await Promise.race([
browser.close(),
new Promise(resolve => setTimeout(resolve, 15000))
])
} catch (_) {}

// We can't wait for the browser to always close cleanly, because it often
// will hang indefinitely. So we just issue the request to close and wait
// 10 sec.
setTimeout(_ => {}, 10000)
}

module.exports = {
Expand Down
4 changes: 3 additions & 1 deletion build.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#!/bin/bash

set -euo pipefail

# This script runs in the docker container, to set up the rust toolchain
# needed to build the adblock-rs node module.

yum install -y openssl-devel nss;
dnf install -y openssl-devel nss;
curl https://sh.rustup.rs -sSf > /tmp/sh.rustup.rs;
sh /tmp/sh.rustup.rs -y;
source ~/.cargo/env;
Expand Down
Loading
Loading