block-dev-tool/server.js

#!/usr/bin/env node

import {PRODUCTION_REGISTRY_URL} from "./env.js";
import path from 'path';
import fetch from "node-fetch";
import express from 'express';
import {create} from 'express-handlebars';
import browserSync from 'browser-sync';
import config from 'config';
import gulp from 'gulp';
import babel from "gulp-babel";
import uglify from "gulp-uglify";
import rename from "gulp-rename";
import dartSass from 'sass';
import gulpSass from 'gulp-sass';
import sourcemaps from "gulp-sourcemaps";
import fs from "fs/promises";
import open from "open";
import {sanitizeUrl} from "@braintree/sanitize-url";
import sanitizeHtml from 'sanitize-html';
import {escape} from "lodash-es";
import {
  getBlockConfigs,
  getBlockData,
  getBlockName,
  getConfigs,
  getImagesList,
  isFileEmpty,
  readJSONFile,
  removeCommentsFromCss,
  removeCommentsFromJs,
  replaceNames,
  uploadFile,
  verifyVersion,
  zipProject
} from "./helpers.js";
import PluginError from 'plugin-error';
import {Server} from "socket.io";
import {createServer} from 'http';
import {authtoken, connect} from "ngrok";
import {exec} from "child_process";
import bodyParser from "body-parser";

/**
 * Constants
 */

const {isDev, modulesPath, projectPath, developmentBlockName} = getConfigs();
const blocksRegistry = isDev ? 'http://localhost:3020' : PRODUCTION_REGISTRY_URL;
const DevToolToken = 'D9lgz0TvzXCnp0xnwVBL109DaAR6Puk6F7YewDhgmP8='; // Temporary token for development purposes.

const dataFiles = await getDataFiles(projectPath);
const DEFAULT_VIEW_LAYOUT = 'alignfull';

/**
 * State
 */

let port = 3000; // This variable is used in `*.hbs` and it will be updated once BrowserSync is ready.
let previewFrameUrl = `/`; // This variable is used in `*.hbs` and it will be updated once BrowserSync is ready.
let shareUrl = '';
const sessions = [];
let ignoreVersionSync = false;
let bs;

/**
 * Init server
 */

const app = express();

// parse application/x-www-form-urlencoded
app.use(bodyParser.urlencoded({extended: false}))

// parse application/json
app.use(bodyParser.json())

const httpServer = createServer(app);
initSessionsServer(httpServer);

const sass = gulpSass(dartSass);

const hbs = create({
  extname: '.hbs', defaultLayout: false, partialsDir: ['.'], helpers: {
    esc_attr(attr) {
      return escape(attr);
    }, esc_url(url) {
      return sanitizeUrl(url);
    }, esc_html(html) {
      // TODO: Check if we can remove this helper.
      return html;
    }, safe_html(html) {
      return sanitizeHtml(html);
    }
  }
});

app.engine('.hbs', hbs.engine);
app.set('view engine', '.hbs');
app.set('views', path.join(modulesPath, 'layouts'));

//
// Routes
//

app.get('/', async (req, res, next) => {
  const data = getBlockConfigs({modulesPath, dataFiles});
  if (data.error && data.errorMessage) {
    return res.send(data.errorMessage);
  }

  const baseView = config.has('baseView') ? config.get('baseView') : DEFAULT_VIEW_LAYOUT;
  const baseViewUrl = `view/${baseView}`;

  data.helpers = {
    include_partial: (filesPath) => handlebarLayoutsPath(modulesPath, filesPath),
  }

  try {
    const verifiedVersion = await verifyVersion(projectPath, blocksRegistry);
    if (!verifiedVersion && !ignoreVersionSync) {
      return res.render('sync', data);
    }
  } catch (err) {
    const errorMessage = "Can't verify block version.";
    console.log(errorMessage, err);
    return next(new Error(errorMessage));
  }

  data.baseView = baseView;
  data.port = `/${baseViewUrl}`;
  data.previewFrameUrl = `${previewFrameUrl}/${baseViewUrl}`;
  data.shareUrl = shareUrl;

  // TODO: Need to review this logic, conflicts with the browsersync work after "/sync" action.
  // if (req.headers.referer) {
  //   // NGROK, public URL
  //   data.shareUrl = undefined; // Link already shared.
  //   data.previewFrameUrl = `/${baseViewUrl}`;
  //   data.publicUrl = true;
  // }

  res.render('index', data);
});

app.get('/view/:baseView', (req, res) => {
  const data = getDataOfFrame(!!req.query.iframe);

  if (data.error && data.errorMessage) {
    return res.send(data.errorMessage);
  }

  const baseView = req.params.baseView ?? DEFAULT_VIEW_LAYOUT;

  res.render(baseView, data)
});

app.get('/publish', async (req, res) => {
  const data = await readJSONFile(path.join(projectPath, `block.json`));

  // Trigger build on the registry server only if the type of the unit is `foundation` or `component`.
  const uploadStaticFiles = ['foundation', 'component'].includes(data.type);

  // Prepare list of static files for the registry server.
  if (uploadStaticFiles) {
    data.static_files = {
      css: getBlockName(data.name).name + '.min.css',
      js: getBlockName(data.name).name + '.min.js',
      images: await getImagesList(path.join(projectPath, 'src', 'images')),
    }

    if (await isFileEmpty(path.join(projectPath, `src/scripts`, data.static_files.js), true)) {
      delete data.static_files.js;
    }

    if (!data.static_files.images.length) {
      delete data.static_files.images;
    }
  }

  let responseData = {
    uploadBundleUrl: undefined,
    staticFilesUrls: {}
  };

  try {
    const response = await fetch(`${blocksRegistry}`, {
      method: 'POST',
      body: JSON.stringify(data),
      headers: {'Content-Type': 'application/json'}
    });

    responseData = await response.json();
  } catch (e) {
    res.json({success: false, message: 'Blocks Registry server is not available.'});
    return;
  }

  if (responseData.statusCode !== 200) {
    res.json({success: false, message: 'Error on registry level.'});
    return;
  }

  // Start files uploading process.
  try {
    if (responseData.uploadBundleUrl) {
      await zipProject(path.join(projectPath, 'src'), path.join(projectPath, 'dist.zip'));
      await uploadFile(path.join(projectPath, 'dist.zip'), responseData.uploadBundleUrl); // Bundle
    }

    if (uploadStaticFiles) {
      if (responseData.staticFilesUrls.css) {
        await uploadFile(
          path.join(projectPath, 'src/styles', data.static_files.css),
          responseData.staticFilesUrls.css.uploadUrl,
          (content) => {
            if (responseData.staticFilesUrls.images) {
              content = replaceNames(content, data.static_files.images, responseData.staticFilesUrls.images);
            }

            removeCommentsFromCss(content)
            return content;
          }); // CSS
      }

      if (responseData.staticFilesUrls.js) {
        await uploadFile(
          path.join(projectPath, 'src/scripts', data.static_files.js),
          responseData.staticFilesUrls.js.uploadUrl,
          (data) => removeCommentsFromJs(data)
        ); // JS
      }

      if (responseData.staticFilesUrls.images) {
        for (let i = 0; i < data.static_files.images.length; i++) {
          await uploadFile(
            path.join(projectPath, 'src/images', data.static_files.images[i]),
            responseData.staticFilesUrls.images[i].uploadUrl,
          ); // Images
        }
      }
    }
  } catch (err) {
    // TODO: Need to update the registry server.
    await fs.unlink(path.join(projectPath, 'dist.zip')); // Remove local bundle

    res.json({success: false, message: err.message});
    return;
  }

  await fs.unlink(path.join(projectPath, 'dist.zip')); // Remove local bundle

  // Trigger project's global files build on the registry server.
  if (uploadStaticFiles) {
    try {
      await triggerGlobalProjectFilesBuild(getBlockName(data.name).project);
    } catch (err) {
      res.json({success: false, message: 'Something wrong with Project Builder.'});
      return;
    }
  }

  res.json({success: true});
});

app.get('/data', async (req, res) => {
  let jsonDataFileName = req.query.name ? req.query.name : 'default';

  const dataFiles = await getDataFiles(projectPath);
  const data = await getBlockData(jsonDataFileName, {projectPath});

  let designPreviewFiles = [];
  try {
    designPreviewFiles = getListOfDesignPreviewFiles(jsonDataFileName, await fs.readdir(path.join(projectPath, 'design', 'preview')));
  } catch (err) {
    console.log('Preview Design doesn\'t exist');
  }

  return res.json({
    dataOptions: dataFiles,
    designPreview: designPreviewFiles,
    data,
  });
});

app.post('/sync', async (req, res) => {
  const blockJson = await readJSONFile(path.join(projectPath, `block.json`));
  const blockName = blockJson.name.startsWith('@') ? blockJson.name : `@${blockJson.name}`;

  if (req.body['ignore']) {
    ignoreVersionSync = true;
    res.json({status: 200, message: 'Version upgrade is ignored.'});
    return;
  }

  bs.pause();

  // Looks like it takes time to pause the browser-sync server, so delay(setTimeout) is necessary.
  setTimeout(() => {
    exec(`node ./node_modules/@axe-web/create-block/create-block.js sync --source ${blockName}`, (err, stdout, stderr) => {
      if (err || stderr) {
        console.error('Error:', err || stderr);
        res.status(500).json({status: 500, message: err || stderr});
      }

      console.log(stdout);

      bs.resume();
      res.json({status: 200, message: 'Successfully synced!'});
    });
  })
});

// Errors handler
app.use(handleSyntaxErrors);

// Static Files
app.use(express.static(path.join(projectPath, 'src')));
app.use(express.static(path.join(projectPath, 'design')));
app.use(express.static(path.join(modulesPath, 'layouts')));

// Custom Middleware
app.use(setHeaders);

// Setup Gulp
await buildAssetFiles();

// BrowserSync
shareUrl = await getShareableUrl();
const bsOptions = await startBrowserSync();
port = bsOptions.port;
previewFrameUrl = bsOptions.previewFrameUrl;
await open(bsOptions.devToolUrl);


//
// Functions
//

function getListOfDesignPreviewFiles(jsonDataFileName, previewFiles) {
  return previewFiles
    .filter(fileName => {
      return fileName.startsWith(jsonDataFileName + '.');
    })
    .map(fileName => {
      const fileData = fileName.split('.');
      const fileFormat = fileData.pop();
      const previewSize = fileData.pop();

      return {
        dataSource: jsonDataFileName,
        widthDimension: Number.parseInt(previewSize, 10),
        url: `/preview/${fileName}`,
      };
    });
}

function getDataOfFrame(isIframe = false) {
  const data = {config: getBlockConfigs({modulesPath, dataFiles})};
  if (data.error && data.errorMessage) {
    return data;
  }

  data.helpers = {
    include_partial: (filesPath) => handlebarLayoutsPath(modulesPath, filesPath),
  }

  if (isIframe) {
    data.iframeMode = true;
  }

  return data;
}

function startBrowserSync() {
  return new Promise((resolve, reject) => {
    const listener = httpServer.listen(0, async () => {
      const PORT = listener.address().port;

      console.log(`The web server has started on port ${PORT}`);

      // BS is global variable.
      bs = browserSync.create();

      const files = getJSBundleFiles();
      gulp.watch(files, {delay: 400}, gulp.series(['build-script-files', function (cb) {
        browserSyncReload(bs, 'js', 'Script Files Change');
        return cb();
      }]));

      gulp.watch(path.posix.join(projectPath, "src/**/*.scss"), {delay: 400}, gulp.series(['build-styling-files', function (cb) {
        browserSyncReload(bs, 'css', 'Style Files Change');
        return cb();
      }]));

      bs.watch(path.join(projectPath, "src/**/*.hbs"), function () {
        return syncTemplate(sessions);
      });

      const args = {
        proxy: `http://localhost:${PORT}`,
        open: false
      };

      if (shareUrl) {
        args.socket = {
          domain: shareUrl
        };
      }

      bs.init(args, (err, bs) => {
        if (err) {
          return reject(err);
        }

        const options = bs.getOptions().toJS();

        const urls = {
          devTool: options.urls.local.replace(options.port, options.proxy.url.port),
          previewFrame: options.urls.local,
        };

        // If local network is available.
        if (options.urls.external) {
          urls.devTool = options.urls.external.replace(options.port, options.proxy.url.port);
          urls.previewFrame = options.urls.external;
        }

        resolve({
          devToolUrl: urls.devTool,
          previewFrameUrl: urls.previewFrame,
          port: options.port
        });
      });
    });
  });
}

function browserSyncReload(bs, extension = '', message = '') {
  if (isDev) {
    // console.log(event, file);
    console.log(message);
  }

  if (extension) {
    extension = "*." + extension;
  }

  bs.reload(extension);
}

function getJSBundleFiles() {
  return [path.posix.join(projectPath, "src/**/*.js"), path.posix.join(projectPath, "src/**/*.mjs"), "!" + path.posix.join(projectPath, "src/**/*.min.js")];
}

function buildScriptFiles(done) {
  const files = getJSBundleFiles();
  return gulp.src(files, {base: path.posix.join(projectPath, 'src')})
    .pipe(sourcemaps.init({}))
    .pipe(babel()).on('error', function (error) {
      showError(new PluginError('JavaScript', error).toString());
      done();
    })
    .pipe(gulp.src(path.join(projectPath, 'vendor/*.js')))
    // .pipe(gulp.dest('src/'))
    .pipe(rename({extname: '.min.js'}))
    .pipe(uglify())
    .pipe(sourcemaps.write('.'))
    .pipe(gulp.dest(path.posix.join(projectPath, 'src')));
}

function buildStyleFiles(done) {
  return gulp.src(path.join(projectPath, 'src/**/*.scss'), {base: path.posix.join(projectPath, 'src')})
    .pipe(sourcemaps.init({}))
    .pipe(sass.sync({outputStyle: 'compressed'}).on('error', function (error) {
      showError(new PluginError('SCSS', error.messageFormatted).toString());
      // sass.logError(error);
      done();
    }))
    // .pipe(gulp.dest('src/'))
    .pipe(rename({extname: '.min.css'}))
    .pipe(sourcemaps.write('.', {}))
    .pipe(gulp.dest(path.posix.join(projectPath, 'src')))
}

function buildAssetFiles() {
  // Register tasks.
  gulp.task('build-script-files', buildScriptFiles);
  gulp.task('build-styling-files', buildStyleFiles);

  // Run first build.
  return new Promise((resolve) => {
    gulp.series('build-script-files', 'build-styling-files', function () {
      resolve();
    })();
  });
}

function showError(errorMessage) {
  console.log(errorMessage);

  // TODO: Send this message to browser.
  // So the developer can understand there is an error.
}

function prepareListOfDataFiles(dataFiles) {
  return dataFiles
    .filter((fileName) => fileName.split('.').pop() === 'json')
    .map((fileName) => {
      const splitName = fileName.split('.');
      splitName.pop();
      return splitName.join('');
    })
    .sort();
}

function handleSyntaxErrors(err, req, res, next) {
  if (err) {
    return res.render('error', {
      helpers: {
        include_partial: (filesPath) => path.join(modulesPath, filesPath),
      },
      err
    });
  }

  next();
}

function handlebarLayoutsPath() {
  return path.join(...arguments)
    .replace(/\\/g, '/'); // Windows path issue. Fix all "\" for Handlebars.
}

function initSessionsServer(httpServer) {
  const io = new Server(httpServer);
  io.on('connection', async (socket) => {
    sessions.push(socket);
    await syncTemplate(sessions);
  });

  return httpServer;
}

function setHeaders(req, res, next) {
  res.setHeader('Access-Control-Allow-Origin', '*');
  next();
}

async function syncTemplate(sessions = []) {
  const blockName = config.has('blockName') ? config.get('blockName') : developmentBlockName;
  const hbsTemplate = await fs.readFile(handlebarLayoutsPath(projectPath, 'src', `${blockName}.template.hbs`), 'utf8');

  sessions.forEach(socket => {
    socket.emit('templateUpdate', {template: hbsTemplate});
  });
}

async function getShareableUrl() {
  let domain = PRODUCTION_REGISTRY_URL;
  let data = {}

  try {
    const response = await fetch(`${domain}/dev-tool/?devToolToken=${DevToolToken}`);
    data = await response.json();
  } catch (e) {
    console.log('Error:', `Can't load data from DevTool endpoint: ${domain}`);
  }

  if (data.statusCode !== 200) {
    console.log('Reply from DevTool endpoint:', data.message);
    return null;
  }

  let url;
  try {
    await authtoken(data.ngrokApiToken);
    url = await connect(port);
  } catch (e) {
    console.log('Error:', `Can't connect to ngrok, probably wrong token.`);
  }

  return url;
}

async function triggerGlobalProjectFilesBuild(project) {
  const response = await fetch(`${blocksRegistry}/project-files`, {
    method: 'POST',
    body: JSON.stringify({project}),
    headers: {'Content-Type': 'application/json'}
  });

  return response.json();
}

async function getDataFiles(projectPath) {
  const dataFiles = [];

  try {
    await fs.access(path.join(projectPath, 'data'));

    const files = prepareListOfDataFiles(await fs.readdir(path.join(projectPath, 'data')));
    dataFiles.push(...files);
  } catch (e) {
    console.log('Warning: data folder not found.');
  }

  return dataFiles;
}