minipack source code parsing and extension

Pre knowledge

  1. First, maybe you need to know what packaging tools exist.
  2. Basic modularization evolution process
  3. Have some knowledge of modular bundle s
  4. Understanding some of babel's common sense
  5. Have some common sense about node

Some common packaging tools

The most common modular building tools today are webpack,rollup,fis,parcel, and so on.

But now the webpack community is relatively large.

In fact, modular development is a big part for program maintainability.

So can we really understand that packaging tools are smart piecing together pieces of modular code? Make our program run normally.

Basic modular evolution

// 1. Global function

function module1 () {
    // do somethings
}
function module2 () {
    // do somethings
}

// 2. Object as a single namespace

var module = {}

module.addpath = function() {}

// 3. IIFE protects private members

var module1 = (function () {
    var test = function (){}
    var dosomething = function () {
        test();
    }
    return {
        dosomething: dosomething
    }
})();

// 4. Multiplexing Module

var module1 = (function (module) {
    module.moduledosomething = function() {}
    return module
})(modules2);

// Then came COMMONJS, AMD, CMD

// node module is a typical COMMONJS

(function(exports, require, module, __filename, __dirname) {
    // The code for the module is actually here
    function test() {
        // dosomethings
    }
    modules.exports = {
        test: test
    }
});

// AMD Asynchronous Load Dependency Pre-position

// RequireeJS example

define('mymodule', ['module depes'], function () {
    function dosomethings() {}
    return {
        dosomethings: dosomethings
    }
})
require('mymodule', function (mymodule) {
    mymodule.dosomethings()
})

// CMD Dependency Postposition 
// seajs example
// mymodule.js
define(function(require, exports, module) {
    var module1 = require('module1')
    module.exports = {
        dosomethings: module1.dosomethings
    }
})

seajs.use(['mymodule.js'], function (mymodule) {
    mymodule.dosomethings();
})


// And the popular esModule

// mymodule 

export default {
    dosomething: function() {}
}

import mymodule from './mymodule.js'
mymodule.dosomething()

Packaging process of minipack

It can be divided into two parts.

  1. Generating module dependencies (unresolved issues such as circular references ~)
  2. Packaging based on processing dependencies

Module Dependency Generation

Specific steps

  1. Given entry file
  2. Analysis of dependencies based on entry files (accessed by bable)
  3. Breadth traversal dependency graph for dependency acquisition
  4. Generate object representation of (module id)key:(array) value based on dependency graph
  5. Establishing require mechanism to realize module loading and running

Source Code Analysis

const fs = require('fs');
const path = require('path');
const babylon = require('babylon');//AST parser
const traverse = require('babel-traverse').default; //Traversal tool
const {transformFromAst} = require('babel-core'); // babel-core

let ID = 0;

function createAsset(filename) {
  const content = fs.readFileSync(filename, 'utf-8');
  // Get the content of the file and do the syntax tree analysis below
  const ast = babylon.parse(content, {
    sourceType: 'module',
  });
  
  // Parse content to AST
  // This array will hold the relative paths of modules this module depends on.
  const dependencies = [];
  // Initialization of dependency sets
  // Using the basics of babel-traverse, you need to find a state and then define it.
  // This is in the Import Declaration state. Then push the dependency value of node import into the dependency set
  traverse(ast, {
    ImportDeclaration: ({node}) => {
      // We push the value that we import into the dependencies array.
      dependencies.push(node.source.value);
    },
  });
  // id self increment
  const id = ID++;

  const {code} = transformFromAst(ast, null, {
    presets: ['env'],
  });

  // Returns all information about such a module
  // The id filename dependency set code we set up
  return {
    id,
    filename,
    dependencies,
    code,
  };
}

function createGraph(entry) {
  // Analytical dependency maps from an entry
  // Start by parsing the entry file.
  const mainAsset = createAsset(entry);

  // Initial dependency set
  const queue = [mainAsset];

  // The common traversal algorithms of a graph are breadth traversal and depth traversal.
  // Here we use breadth traversal
  for (const asset of queue) {
    // mapping records for current dependencies
    asset.mapping = {};
    // Get the dependency module address
    const dirname = path.dirname(asset.filename);
    // There was only one asset at first, but dependencies could be multiple
    asset.dependencies.forEach(relativePath => {
      // Here we get the absolute path.
      const absolutePath = path.join(dirname, relativePath);
      // Here's an analysis.
      // The analytic equivalent of this layer diffuses to the next layer and traverses the entire graph.
      const child = createAsset(absolutePath);

      // Equivalent to associating the current module with the sub-module
      asset.mapping[relativePath] = child.id;
      // Breadth traversal with queues
      queue.push(child);
    });
  }

  // Return to a queue that has traversed the dependencies
  return queue;
}
function bundle(graph) {
  let modules = '';
  graph.forEach(mod => {
    modules += `${mod.id}: [
      function (require, module, exports) { ${mod.code} },
      ${JSON.stringify(mod.mapping)},
    ],`;
  });
  // Common JS Style
  const result = `
    (function(modules) {
      function require(id) {
        const [fn, mapping] = modules[id];
        function localRequire(name) {
          return require(mapping[name]);
        }
        const module = { exports : {} };
        fn(localRequire, module, module.exports);
        return module.exports;
      }

      require(0);
    })({${modules}})
  `;
  return result;
}

A simple example

// doing.js 
import t from './hahaha.js'

document.body.onclick = function (){
    console.log(t.name)
}

// hahaha.js

export default {
    name: 'ZWkang'
}

const graph = createGraph('../example/doing.js');
const result = bundle(graph);

The example result is as follows

// The packaged code is similar
    (function(modules) {
      function require(id) {
        const [fn, mapping] = modules[id];
        function localRequire(name) {
          return require(mapping[name]);
        }
        const module = { exports : {} };
        fn(localRequire, module, module.exports);
        return module.exports;
      }

      require(0);
    })({0: [
      function (require, module, exports) { "use strict";
        
        var _hahaha = require("./hahaha.js");
        
        var _hahaha2 = _interopRequireDefault(_hahaha);
        
        function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
        
        document.body.onclick = function () {
          console.log(_hahaha2.default.name);
        }; },
      {"./hahaha.js":1},
    ],1: [
      function (require, module, exports) { "use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = {
  name: 'ZWkang'
}; },
      {},
    ],})
Dependent graph-generated files can be simplified to
modules = {
    0: [function code , {deps} ],
    1: [function code , {deps} ]
}

require simulates the operation of a very simple COMMONJS module

function require(id) {
    const [fn, mapping] = modules[id];
    function localRequire(name) {
      return require(mapping[name]);
    }
    const module = { exports : {} };
    fn(localRequire, module, module.exports);
    return module.exports;
}

require(0);

Analyzed

Our module code will be executed. And the results of execution are stored in module.exports

And accepts three parameters: require module.exports.

COMMONJS module like this injects exports, require, module, _filename, _dirname into the module closure

It require s its code at the entry.

minipack source summary

Through the above analysis, we can understand that

  • The basic structure of minipack
  • Basic Forms of Packaging Tools
  • Some Problems of Modules

extend

Now that bundle s have been implemented, can we implement a simple HMR for hot replacement module content based on minipack?

It can be implemented simply.

A Simple HMR Implementation

It can be divided into the following steps

  1. watch file change
  2. emit update to front-end
  3. front-end replace modules

Of course, there are more careful handling.

For example, module subdivision hotload processing, HMR granularity, etc.

It's also important to consider when setting up module bundle s.

Implementation based on minipack

We can imagine what needs to be done.

Changes in watch module asset
Use ws for front-end and back-end update notification.
Change front-end modules [change id]

// Create a folder directory format as follows

- test.js
- base.js
- bundle.js
- wsserver.js
- index.js
- temp.html
// temp.html

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta http-equiv="X-UA-Compatible" content="ie=edge">
    <title>Document</title>
</head>
<body>
    <button class="click"> click me </button>
    <% script %> 
    <!-- Replacement placeholder -->
</body>
</html>
// base.js and test.js are the modules for testing.
// base.js

var result = {
    name: 'ZWKas'
}

export default result

// test.js

import t from './base.js'

console.log(t, '1');
document.body.innerHTML = t.name

Changes in watch module asset

// First is the first step.
// watch asset file

function createGraph(entry) {
  // Start by parsing the entry file.
  const mainAsset = createAsset(entry);

  const queue = [mainAsset];

  for (const asset of queue) {
    asset.mapping = {};

    const dirname = path.dirname(asset.filename);

    fs.watch(path.join(__dirname,asset.filename), (event, filename) => {
        console.log('watch ',event, filename)
        const assetSource = createAsset(path.join(__dirname,asset.filename))
        wss.emitmessage(assetSource)
    })
    asset.dependencies.forEach(relativePath => {

      const absolutePath = path.join(dirname, relativePath);

      const child = createAsset(absolutePath);

      asset.mapping[relativePath] = child.id;
      queue.push(child);
    });
  }

  return queue;
}

Simply modified createGraphl to add fs.watch method as trigger point.

(watch events may trigger several times depending on the underlying implementation triggered by the operating system)

While creating the resource map, the resource is watch ed.

There's one more thing to add here. When we use creareAsset, if we do not associate id with path, then the id that is triggered again will change.

You can directly associate absolute addresses with module id s. Thus, the id of module is reused.

// createasset Some Code Changes Key Codes
let mapWithPath = new Map()
if(!mapWithPath.has(path.resolve(__dirname, filename))) {
    mapWithPath.set(path.resolve(__dirname, filename), id)
}
const afterid = mapWithPath.get(path.resolve(__dirname, filename))
return {
    id: afterid,
    filename,
    dependencies,
    code,
};

Interactive prompt update using websockt

 
// wsserver.js file is the second step. Use websocket to interact with the front end and prompt update


const EventEmitter = require('events').EventEmitter
const WebSocket = require('ws')

class wsServer extends EventEmitter {
    constructor(port) {
        super()
        this.wss = new WebSocket.Server({ port });
        this.wss.on('connection', function connection(ws) {
            ws.on('message', function incoming(message) {
              console.log('received: %s', message);
            });
        });
    }
    emitmessage(assetSource) {
        this.wss.clients.forEach(ws => {
            ws.send(JSON.stringify({
                type: 'update',
                ...assetSource
            }))
        })
    }
}


const wsserver = new wsServer(8080)
module.exports = wsserver
// Simply export a web socket instance with update information transmitted to client

Triggered at fs.watch trigger point


const assetSource = createAsset(path.join(__dirname,asset.filename))
wss.emitmessage(assetSource)

This is what we do here. Re-create the resource map. Including id,code, etc.

bundle.js is our packaging operation

const minipack = require('./index')
const fs = require('fs')

const makeEntry = (entryHtml, outputhtml ) => {
    const temp = fs.readFileSync(entryHtml).toString()
    // console.log(temp)caches.c
    const graph = minipack.createGraph('./add.js')

    const result = minipack.bundle(graph)

    const data = temp.replace('<% script %>', `<script>${result}</script><script>
    const ws = new WebSocket('ws://127.0.0.1:8080')

    ws.onmessage = function(data) {
        console.log(data)
        let parseData
        try {
            parseData = JSON.parse(data.data)
        }catch(e) {
            throw e;
        }
        if(parseData.type === 'update') {
            const [fn,mapping] = modules[parseData.id]
            modules[parseData.id] = [
                new Function('require', 'module', 'exports', parseData.code),
                mapping
            ]
            require(0)
        }
    }
    
    </script>`)
    fs.writeFileSync(outputhtml, data)
}

makeEntry('./temp.html', './index.html')

The operation is to get temp.html and package dependency graphs into script in temp.html

And establish the ws link. To obtain data

Module replacement at the front end

const [fn,mapping] = modules[parseData.id]
modules[parseData.id] = [
    new Function('require', 'module', 'exports', parseData.code),
    mapping
] // Here's how to refresh the corresponding module
require(0) // Rerun once from the entrance

Of course, some meticulous operations may replace only the referenced module parent, but here's a simplified version that you can do without first.

Then we go to the file of run bundle.js and we will find that the watch mode is turned on. here
Access the generated index.html file

When we change the content of base.js




So a simple HMR based on minipack is completed.

But obviously, there are many problems. Pure as throwing bricks to attract jade.

(For example, the side effects of module, resources only have js resources and so on, there are many interesting points in careful analysis.)

Extended reading

Sample code for this article

minipack hmr

Contact me

Tags: node.js JSON Webpack github IE

Posted on Mon, 22 Apr 2019 15:51:34 -0700 by jackiw