Scraper Studio IDE 最佳实践

try {
  // 即使页面已无效，也会等待 30 秒查找 'ok-selector'
  wait('ok-selector');
} catch (e) {
  // 仅凭 wait 超时无法证明页面无效
  dead_page("Page doesn't exist");
}

wait('ok-selector, 404-selector');
if (el_exists('404-selector'))
  dead_page();

if (!(el_exists('#price1')) || el_exists('#price2')
  || el_exists('#price3') || el_exists('#discount'))
{
    dead_page('No price found');
}

if (!el_exists('#price1, #price2, #price3, #discount'))
  dead_page('No price found');

navigate(input.url);
let $ = load_html(html());
let next_page_url = $('.next_page').attr('href');
rerun_stage({url: next_page_url});

let url = new URL(input.url);
if (input.page)
    url.searchParams.set('page', input.page);
navigate(url);
// 只有当本阶段被针对某一页重新运行时，input.page 才存在。
// 根页面上 input.page 为 undefined，逻辑会继续向下分发。
if (input.page)
    return;

let $ = load_html(html());
let total_products = +$('.total_pages').text();
let total_pages = Math.ceil(total_products / 20);
total_pages = Math.min(total_pages, 50);

for (let page = 2; page <= total_pages; page++)
    rerun_stage({url: input.url, page});

navigate('https://example.com');
try {
  wait_visible('.cky-btn-accept', {timeout: 5000});
  click('.cky-btn-accept');
} catch (e) {
    console.log('Accept cookies button does not exist, continue');
}

// 在后台运行，不会给每一步增加延迟。
// 监视器会在每次交互前自动检查弹窗。
close_popup('.cky-btn-accept', '.cky-btn-accept');
navigate('https://example.com');
click('.open-product-full-info');

tag_response('product', /api\/product/);
navigate('https://example.com');

// 解析器代码：
// 请求可能尚未完成，product 可能是 undefined
let {product} = parser;
return product.data;

tag_response('product', /api\/product/);
navigate('https://example.com');
wait_for_parser_value('product');

// 解析器代码：
let {product} = parser;
return product.data;

tag_response('product', /api\/product/);
navigate('https://example.com');

// wait_for_parser_value 会返回值，方便在交互代码中使用
let product = wait_for_parser_value('product');
navigate(product.reviews_url);
tag_html('reviews_html');

// 解析器代码：
let {product, reviews_html} = parser;
let $ = load_html(reviews_html);
let reviews = $('.review').toArray().map(v => $(v).text());
return {
  ...product.data,
  reviews,
};

try {
  wait('selector1');
  // some code
  wait('selector2');
  // some code
} catch (e) {
  throw "Page not loaded properly"
}

// Crawler error: waiting for selector "selector1" failed: timeout 30000ms exceeded
wait('selector1');
// some code
wait('selector2');
// some code

if (!el_exists('.product-title'))
    throw new Error('Failed to load product page');

// 120 秒太长；平台无法回收卡住的 peer
wait('selector', {timeout: 120000});

wait('selector');                       // 默认 30 秒
wait('selector', {timeout: 45000});     // 略慢的页面用 45 秒
wait('selector', {timeout: 60000});     // 长期较慢的页面用 60 秒

let counter = input.counter || 5;
while (counter > 1) {
  try {
    wait('selector', {timeout: 500});
    click('selector');
    type('selector');
    // some code
    break;
  } catch (e) {
    // rerun_stage 会创建新会话，但这种模式会消耗额外 CPM
    return rerun_stage({...input, counter: --counter});
  }
}

navigate('https://example.com');
wait('h1');

try {
  const example = obj.prop;
} catch (e) {}

// 无谓地浪费浏览器时间
try { wait('selector'); } catch (e) {}
try { wait_network_idle({timeout: 8000}); } catch (e) {}
try { wait_page_idle(); } catch (e) {}

const example = object?.prop;
const example2 = object.prop ?? undefined;
const example3 = object.prop ? object.prop : undefined;

const links = [];
$('.card.product-wrapper').each(function (i, el) {
  links.push({url: $(this).find('h4 a').attr('href')});
})
return links;

const links = $('.card.product-wrapper').toArray().map(v => ({
  url: $(v).find('h4 a').attr('href'),
}));

$.prototype.clearText = function () {
  return this.text().replace(/\s+/g, ' ').trim();
}

let name = $('a').text_sane();

// 仅提取数字时：
let value = +$('a').text().replace(/\D+/g, '');