/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ // TODO: Cases that needs lots of network loading still may fail. Like bmap-xxx const puppeteer = require('puppeteer'); const slugify = require('slugify'); const fse = require('fs-extra'); const fs = require('fs'); const path = require('path'); const program = require('commander'); const compareScreenshot = require('./compareScreenshot'); const {testNameFromFile, fileNameFromTest, getVersionDir, buildRuntimeCode, getEChartsTestFileName, waitTime} = require('./util'); const {origin} = require('./config'); const cwebpBin = require('cwebp-bin'); const { execFile } = require('child_process'); const {runTasks} = require('./task'); const chalk = require('chalk'); // Handling input arguments. program .option('-t, --tests ', 'Tests names list') .option('--no-headless', 'Not headless') .option('-s, --speed ', 'Playback speed') .option('--expected ', 'Expected version') .option('--expected-source ', 'Expected source') .option('--actual ', 'Actual version') .option('--actual-source ', 'Actual source') .option('--renderer ', 'svg/canvas renderer') .option('--use-coarse-pointer ', '"auto" (by default) or "true" or "false"') .option('--threads ', 'How many threads to run concurrently') .option('--no-save', 'Don\'t save result') .option('--dir ', 'Out dir'); program.parse(process.argv); program.speed = +program.speed || 1; program.actual = program.actual || 'local'; program.threads = +program.threads || 1; program.renderer = (program.renderer || 'canvas').toLowerCase(); program.useCoarsePointer = (program.useCoarsePointer || 'auto').toLowerCase(); program.dir = program.dir || (__dirname + '/tmp'); if (!program.tests) { throw new Error('Tests are required'); } if (!program.expected) { throw new Error('Expected version is required'); } console.log('Playback Ratio: ', program.speed); function getScreenshotDir() { return `${program.dir}/__screenshot__`; } function sortScreenshots(list) { return list.sort((a, b) => { return a.screenshotName.localeCompare(b.screenshotName); }); } function getClientRelativePath(absPath) { return path.join('../', path.relative(__dirname, absPath)); } function replaceEChartsVersion(interceptedRequest, source, version) { // TODO Extensions and maps if (interceptedRequest.url().endsWith('dist/echarts.js')) { console.log('Use echarts version: ' + source + ' ' + version); interceptedRequest.continue({ url: `${origin}/test/runTest/${getVersionDir(source, version)}/${getEChartsTestFileName()}` }); } else { interceptedRequest.continue(); } } async function convertToWebP(filePath, lossless) { const webpPath = filePath.replace(/\.png$/, '.webp'); return new Promise((resolve, reject) => { execFile(cwebpBin, [ filePath, '-o', webpPath, ...(lossless ? ['-lossless'] : ['-q', 75]) ], (err) => { if (err) { reject(err); } else { resolve(webpPath); } }); }); } async function takeScreenshot(page, fullPage, fileUrl, desc, isExpected, minor) { let screenshotName = testNameFromFile(fileUrl); if (desc) { screenshotName += '-' + slugify(desc, { replacement: '-', lower: true }); } if (minor) { screenshotName += '-' + minor; } const screenshotPrefix = isExpected ? 'expected' : 'actual'; fse.ensureDirSync(getScreenshotDir()); const screenshotPath = path.join(getScreenshotDir(), `${screenshotName}-${screenshotPrefix}.png`); await page.screenshot({ path: screenshotPath, // https://github.com/puppeteer/puppeteer/issues/7043 // https://github.com/puppeteer/puppeteer/issues/6921#issuecomment-829586680 captureBeyondViewport: false, fullPage }); let webpScreenshotPath; try { webpScreenshotPath = await convertToWebP(screenshotPath); } catch (e) { console.error('Failed to convert screenshot to webp', e); } console.log('Screenshot: ', webpScreenshotPath || screenshotPath); return { screenshotName, screenshotPath: webpScreenshotPath || screenshotPath, rawScreenshotPath: screenshotPath }; } async function waitForNetworkIdle(page) { let count = 0; const started = () => (count = count + 1); const ended = () => (count = count - 1); page.on('request', started); page.on('requestfailed', ended); page.on('requestfinished', ended); return async (timeout = 5000) => { while (count > 0) { await waitTime(100); if ((timeout = timeout - 100) < 0) { console.error('Timeout'); } } page.off('request', started); page.off('requestfailed', ended); page.off('requestfinished', ended); }; } /** * @param {puppeteer.Browser} browser */ async function runTestPage(browser, testOpt, source, version, runtimeCode, isExpected) { const fileUrl = testOpt.fileUrl; const screenshots = []; const logs = []; const errors = []; const page = await browser.newPage(); page.setRequestInterception(true); page.on('request', request => replaceEChartsVersion(request, source, version)); async function pageScreenshot() { if (!program.save) { return; } // Final shot. await page.mouse.move(0, 0); const desc = 'Full Shot'; const { screenshotName, screenshotPath, rawScreenshotPath } = await takeScreenshot(page, true, fileUrl, desc, isExpected); screenshots.push({ screenshotName, desc, screenshotPath, rawScreenshotPath }); } let vstInited = false; await page.exposeFunction('__VRT_INIT__', () => { vstInited = true; }); await page.exposeFunction('__VRT_MOUSE_MOVE__', async (x, y) => { await page.mouse.move(x, y); }); await page.exposeFunction('__VRT_MOUSE_DOWN__', async () => { await page.mouse.down(); }); await page.exposeFunction('__VRT_MOUSE_UP__', async () => { await page.mouse.up(); }); await page.exposeFunction('__VRT_LOAD_ERROR__', async (err) => { errors.push(err); }); // await page.exposeFunction('__VRT_WAIT_FOR_NETWORK_IDLE__', async () => { // await waitForNetworkIdle(); // }); // TODO should await exposeFunction here const waitForScreenshot = new Promise((resolve) => { page.exposeFunction('__VRT_FULL_SCREENSHOT__', async () => { await pageScreenshot(); resolve(); }); }); const waitForActionFinishManually = new Promise((resolve) => { page.exposeFunction('__VRT_FINISH_ACTIONS__', async () => { resolve(); }); }); page.exposeFunction('__VRT_LOG_ERRORS__', (err) => { errors.push(err); }); let actionScreenshotCount = {}; await page.exposeFunction('__VRT_ACTION_SCREENSHOT__', async (action) => { if (!program.save) { return; } const desc = action.desc || action.name; actionScreenshotCount[action.name] = actionScreenshotCount[action.name] || 0; const { screenshotName, screenshotPath, rawScreenshotPath } = await takeScreenshot(page, false, testOpt.fileUrl, desc, isExpected, actionScreenshotCount[action.name]++); screenshots.push({ screenshotName, desc, screenshotPath, rawScreenshotPath }); }); await page.evaluateOnNewDocument(runtimeCode); page.on('console', msg => { // console.log('Page Log: ', msg.text()); logs.push(msg.text()); }); page.on('pageerror', error => { console.error('Page Error: ', error.toString()); errors.push(error.toString()); }); page.on('dialog', async dialog => { await dialog.dismiss(); }); try { await page.setViewport({ width: 800, height: 600 }); await page.goto(`${origin}/test/${fileUrl}?__RENDERER__=${program.renderer}&__COARSE__POINTER__=${program.useCoarsePointer}`, { waitUntil: 'networkidle2', timeout: 10000, // timeout: 0 }); if (!vstInited) { // Not using simpleRequire in the test console.log(`Automatically started in ${testNameFromFile(fileUrl)}`); await page.evaluate(() => { __VRT_START__(); }); } // Wait do screenshot after inited await waitForScreenshot; let actions = []; try { let actContent = await fse.readFile(path.join(__dirname, 'actions', testOpt.name + '.json')); actions = JSON.parse(actContent); } catch (e) { // console.log(e); } if (actions.length > 0) { try { page.evaluate((actions) => { __VRT_RUN_ACTIONS__(actions); }, actions); } catch (e) { errors.push(e.toString()); } // We need to use the actions finish signal if there is reload happens in the page. // Because the original __VRT_RUN_ACTIONS__ not exists anymore. await waitForActionFinishManually; } } catch(e) { console.error(e); } await page.close(); return { logs, errors, screenshots: screenshots }; } function writePNG(diffPNG, diffPath) { return new Promise((resolve, reject) => { const writer = fs.createWriteStream(diffPath); diffPNG.pack().pipe(writer); writer.on('finish', resolve); writer.on('error', reject); }); }; /** * @param {puppeteer.Browser} browser */ async function runTest(browser, testOpt, runtimeCode, expectedSource, expectedVersion, actualSource, actualVersion) { if (program.save) { testOpt.status === 'running'; const expectedResult = await runTestPage(browser, testOpt, expectedSource, expectedVersion, runtimeCode, true); const actualResult = await runTestPage(browser, testOpt, actualSource, actualVersion, runtimeCode, false); // sortScreenshots(expectedResult.screenshots); // sortScreenshots(actualResult.screenshots); const screenshots = []; let idx = 0; for (let shot of expectedResult.screenshots) { const expected = shot; const actual = actualResult.screenshots[idx++]; const result = { actual: getClientRelativePath(actual.screenshotPath), expected: getClientRelativePath(expected.screenshotPath), name: actual.screenshotName, desc: actual.desc }; try { const {diffRatio, diffPNG} = await compareScreenshot( expected.rawScreenshotPath, actual.rawScreenshotPath ); const diffPath = `${getScreenshotDir()}/${shot.screenshotName}-diff.png`; await writePNG(diffPNG, diffPath); let diffWebpPath; try { diffWebpPath = await convertToWebP(diffPath); } catch (e) { console.error('Failed to convert diff png to webp', e); } result.diff = getClientRelativePath(diffWebpPath || diffPath); result.diffRatio = diffRatio; // Remove png files try { await Promise.all([ fse.unlink(actual.rawScreenshotPath), fse.unlink(expected.rawScreenshotPath), diffWebpPath && fse.unlink(diffPath) ]); } catch (e) {} } catch(e) { result.diff = ''; result.diffRatio = 1; console.log(e); } screenshots.push(result); } testOpt.results = screenshots; testOpt.status = 'finished'; testOpt.actualLogs = actualResult.logs; testOpt.expectedLogs = expectedResult.logs; testOpt.actualErrors = actualResult.errors; testOpt.expectedErrors = expectedResult.errors; testOpt.actualVersion = actualVersion; testOpt.expectedVersion = expectedVersion; testOpt.useSVG = program.renderer === 'svg'; testOpt.useCoarsePointer = program.useCoarsePointer; testOpt.lastRun = Date.now(); } else { // Only run once await runTestPage(browser, testOpt, 'local', runtimeCode, true); } } async function runTests(pendingTests) { const browser = await puppeteer.launch({ headless: program.headless, args: [`--window-size=830,750`] // new option }); // TODO Not hardcoded. // let runtimeCode = fs.readFileSync(path.join(__dirname, 'tmp/testRuntime.js'), 'utf-8'); let runtimeCode = await buildRuntimeCode(); runtimeCode = `window.__VRT_PLAYBACK_SPEED__ = ${program.speed || 1};\n${runtimeCode}`; process.on('exit', () => { browser.close(); }); async function eachTask(testOpt) { console.log(`Running test: ${testOpt.name}, renderer: ${program.renderer}, useCoarsePointer: ${program.useCoarsePointer}`); try { await runTest(browser, testOpt, runtimeCode, program.expectedSource, program.expected, program.actualSource, program.actual); } catch (e) { // Restore status testOpt.status = 'unsettled'; console.error(e); } if (program.save) { process.send(testOpt); } } // console.log('Running threads: ', program.threads); // await runTasks(pendingTests, async (testOpt) => { // await eachTask(testOpt); // }, program.threads); try { for (let testOpt of pendingTests) { await eachTask(testOpt); } } catch(e) { console.error(e); } await browser.close(); } runTests(program.tests.split(',').map(testName => { return { fileUrl: fileNameFromTest(testName), name: testName, results: [], actualLogs: [], expectedLogs: [], actualErrors: [], expectedErrors: [], status: 'pending' }; }));