TempAccess Auth Portal

TempAccess was a side company that I created with the purpose of providing temporary login credentials to any site. In my original implementation I created webapp for users to store their passwords and their associated url and TempAccess would generate a 'proxy' password. They could then give this proxy password to a contractor, friend, employee, etc. who could navigate to the url and use the TempAccess chrome extension to log in using the proxy password instead of the real one.

The problem with this approach was security. With this implementation you had to replace the proxy password with the real password on the user's browser and anytime you are sending information to a client, it is exposed. Even with encryption, at some point it had to be unencrypted to be able to be sent to the site's authentication.

The solution I came up with was instead of the user logging in on their browser, I instead open a headless browser in a google cloud server and screencast the frames back to the user. This way the user thinks they're logging in normally, but they just have an image of whats going on instead of having access to the javascript behind the login.

This snippet has 3 parts:
1. The function that handles the password field locating and replacing logic.
2. A function that extracts the relevant cookies from the headless browser session to be sent back to the user.
3. A function to handle user input as they execute mouse moves, clicks, typing, etc.


private async attachSubmitListener() {
    await this._page.evaluate((proxyPassword) => {

        const extractBaseUrl = (url: string) => {
            const pathArray = url.split( '/' );
            const protocol = pathArray[0];
            const host = pathArray[2];
            return protocol + '//' + host;
        }

        const findSubmitElem = (element: HTMLElement, depth=0): HTMLElement | undefined => {
            // Check upwards first.
            if (depth <= 0 && depth > -4) {
                if (element.parentElement) {
                    const foundSubmitElem: HTMLElement | undefined = findSubmitElem(element.parentElement, depth - 1);
                    if (foundSubmitElem) return foundSubmitElem;
                }
            }
            // Check this element.
            if (element.localName === 'button') return element;
            for (let i = 0; i < element.attributes.length; i++) {
                const attr = element.attributes[i];
                if (attr.value.includes('submit')) return element;
            }
            if (element.childElementCount === 0) {
                // We've reached the bottom, check innerHTML.
                const innerHTML = element.innerHTML.toLowerCase();
                if (innerHTML.includes('next') || innerHTML.includes('in')) return element;
            } else if (depth >= 0 && element.childNodes) { // Dont drill down a node that is upstream.
                // Drill down
                for (let i = 0; i < element.childNodes.length; i++) {
                    const foundSubmitElem: HTMLElement | undefined = findSubmitElem(element.childNodes[i] as HTMLElement, depth + 1);
                    if (foundSubmitElem) return foundSubmitElem;
                }
            }
            return;
        }

        /**
         * Checks if the current page url matches the proxyPass and if it is prior to the expration date. If it is all good,
         * the passwordInputEleent value will be replaced with the real password and relevant events will be dispatched.
         *
         * @returns True if the password was replaced, else false.
         */
        const handleSubmit = async () => {
            // Wait for proxyPass promise to resolve. This allows listeners to be set before we've retrieved the proxyPass.
            const proxyPass = await window.getProxyPass();
            if (proxyPass === null || !window.tempAccess.passwordInputElement) return;

            const baseUrl = extractBaseUrl(window.location.href);
            const domainPattern = `^https?://([a-z0-9]+[.])?${proxyPass.domain}$` // Allows any subdomain
            const domainRegex = new RegExp(domainPattern, "g");

            // Test that the proxy password matches the requested domain
            if (domainRegex.test(baseUrl)) {
                if (proxyPass.expires === null || new Date().getTime() < proxyPass.expires) {

                    window.setShouldCast(false);
                    window.tempAccess.passwordInputElement.focus();
                    const textLength = window.tempAccess.passwordInputElement.value.length;
                    window.tempAccess.passwordInputElement.value = '';
                    await window.typeText(proxyPass.password, textLength)
                    return true;
                } else {
                    window.output("ERROR", `This proxy password expired at ${new Date(proxyPass.expires)}`);
                }
            } else {
                window.output('ERROR', `Password domain ${proxyPass.domain} does not match site ${baseUrl}`)
            }
            return false;
        }

        const killEvent = (e: Event) => {
            e.preventDefault();
            e.stopPropagation();
            e.stopImmediatePropagation();
        }

        /**
         * If the proxyPassword was replaced, stop screencasting temporarily.
         */
        const onSuccess = async () => {
            window.tempAccess.passwordInputElement.focus();
            const textLength = window.tempAccess.passwordInputElement.value.length;
            window.tempAccess.passwordInputElement.value = '';
            await window.typeText(proxyPassword, textLength);
            window.tempAccess.passwordInputElement.blur();
            window.setShouldCast(true);
        }

        /**
         * Event handler for click events that triggers handleSubmit.
         *
         * TODO: This should only be fired on elements that may indicate an actual submit.
         *
         * @param e
         * @returns false (neccessary for capture)
         */
        const clickHandler = async (e: MouseEvent) => {
            const submitElem = findSubmitElem(e.target as HTMLElement);
            let success: boolean | undefined;
            if (submitElem && proxyPassword) {
                killEvent(e);
                success = await handleSubmit();
                submitElem.click();
            }
            setTimeout(async () => {
                if (success) await onSuccess();
                document.addEventListener('click', clickHandler, { once: true, capture: true });
            }, 1500)
            return false;
        }

        /**
         * Event handler for keyboard events with Enter key (submits form)
         *
         * @param e
         * @returns false (neccessary for capture)
         */
        const enterHandler = async (e: KeyboardEvent) => {
            let success: boolean | undefined;
            if (e.key === 'Enter') {
                killEvent(e);
                success = await handleSubmit();
                document.dispatchEvent(new KeyboardEvent('keypress', { key: 'Enter', bubbles: true, cancelable: true }));
            }
            setTimeout(async () => {
                if (success) await onSuccess();
                document.addEventListener('keypress', enterHandler, { once: true, capture: true });
            }, 1500)
            return false
        }

        // Mouseup and keyup so first run to run after focusout
        // document.removeEventListener('click', clickHandler);
        document.addEventListener('click', clickHandler, { once: true, capture: true });
        // document.removeEventListener('keypress', enterHandler);
        document.addEventListener('keypress', enterHandler, { once: true, capture: true });
    }, this._proxyPassword)
}

/**
 * Authentication either works by cookies or tokens. We cant do anything about token auth (yet)
 * but when a login is successful, extract all cookies (because we cant determine the correct auth cookies)
 * and send them to the frontend to be set.
 *
 * Cookies can ONLY be set by the domain they are associated for, thats why we need to use a chrome extension;
 * to set cookies.
 *
 * @returns The page cookies from the puppetter page.
 */
private async extractCookies() {
    const cookies = (await this._session.send('Network.getCookies')).cookies;
    const url = extractBaseUrl(this._authUrl);
    const chromeCookies = cookies.map((cookie: any) => {
        let sameSite;
        if (cookie.sameSite) {
            sameSite = `${cookie.sameSite}`.toLowerCase()
            if (sameSite === 'none') {
                sameSite = 'no_restriction'
            }
        }
        let domain = `${cookie.domain}`;
        if (domain.startsWith('.')) domain = domain.slice(1)
        return {
            name: cookie.name,
            value: cookie.value,
            path: cookie.path,
            expirationDate: cookie.expires,
            httpOnly: cookie.httpOnly,
            secure: cookie.secure,
            ...(!cookie.name.startsWith('__Host-') && { domain: cookie.domain }),
            ...(sameSite && { sameSite }),
            url: `https://${domain}${cookie.path}`
        }
    })
    console.log(`[${this._portalId}]  ${chromeCookies.length} Cookies`)
    return chromeCookies;
}

/**
 * Handle input commands from the frontend. These commands
 * are explicit to make sure generic events
 * cannot be submitted to the client.
 *
 * @param command
 * @param data
 */
private async onCommand({ command, data }: { command: ProtocolCommand, data: any }) {
    if (!this._isRunning || !this._shouldCast) return;
    if (this._isTyping) {
        await this.waitForTyping();
    }
    switch (command) {
        case InputCommand.dispatchKeyEvent:
        case InputCommand.dispatchMouseEvent:
        case PageCommand.dispatchReload:
            if (this._session && this._session.connection()) {
                this._session.send(command, data);
            }
            break;

        case InputCommand.dispatchPasteEvent:
            this.typeText(data.text);
            break;

        case BrowserCommand.dispatchSetViewport:
            this._height = data.height;
            this._width = data.width;
            this._page && this._page.setViewport(data);
            break;

        case BrowserCommand.dispatchClose:
            this.exit();
            break;

        case BrowserCommand.dispatchCloseWithAuth:
            this.closeWithAuth();
            break;

    }
}
Determine Pointcloud Bounding Box

Finding the bounding box of a pointcloud, which is just a collection of points in 3D space, is a trivial task: just take the minimum and maximum values along each axis. However, finding the bounding box of a point cloud where need to find the smallest bounding box of the point cloud while allowing the bounding box to be rotated in 3D space is a more difficult task.

This algorithm first uses Principal Component Analysis to reduce the dimensionality of the point cloud. It then iterates over all the edges of the convex hull of the 2 dimensional cloud and fits a bounding box to the point cloud aligned along that edge. It keeps track of the bounding box with the minimum volume as the optimal solution. The insight behind this algorithm is the assumption that the smallest bounding box will likely be aligned along one of the edges of this convex hull.


def get_bounding_box(self, cloud):
    """
    Finds vertices of the rotated bounding box of a point cloud cloud.
    Parameters
    ----------
    cloud: [n,3] ndarray
        Point cloud.
    Return
    ------
    vertices: [8,3] ndarray
        Coordinates for vertices of the object bouding box.\n
        [min_x,min_y,max_z]\n
        [max_x,min_y,max_z]\n
        [min_x,max_y,max_z]\n
        [max_x,max_y,max_z]\n
        [min_x,min_y,min_z]\n
        [max_x,min_y,min_z]\n
        [min_x,max_y,min_z]\n
        [max_x,max_y,min_z]
    dimensions: [3,] ndarray
        The dimensions of the bounding box (lenght,width,height) where length > width.
    """

    pca = PCA(n_components=3)
    pca_cloud = pca.fit_transform(cloud)
    hull = ConvexHull(pca_cloud[:, :2])

    max_z = pca_cloud[:, 2].min()
    min_z = pca_cloud[:, 2].max()
    if pca.components_[2, 2] < 0:
        direction = -1
        max_z = pca_cloud[:, 2].max()
        min_z = pca_cloud[:, 2].min()

    volume_bb = 10000000
    for simplex in hull.simplices:
        u = pca_cloud[simplex[0], :2]
        v = pca_cloud[simplex[1], :2]
        slope = (v[1] - u[1])/(v[0] - u[0])
        theta = np.arctan(slope)
        rot = np.array([[np.cos(theta), -np.sin(theta), 0.0],
                        [np.sin(theta), np.cos(theta), 0.0], [0.0, 0.0, 1.0]])
        pc = pca_cloud.dot(rot)
        min_x = pc[:, 0].min()
        max_x = pc[:, 0].max()
        min_y = pc[:, 1].min()
        max_y = pc[:, 1].max()
        vol = np.abs(min_x-max_x)*np.abs(min_y-max_y)
        if vol < volume_bb:
            vertices = np.zeros((8, 3))
            vertices[0, :] = [min_x, min_y, max_z]
            vertices[1, :] = [max_x, min_y, max_z]
            vertices[2, :] = [min_x, max_y, max_z]
            vertices[3, :] = [max_x, max_y, max_z]
            vertices[4, :] = [min_x, min_y, min_z]
            vertices[5, :] = [max_x, min_y, min_z]
            vertices[6, :] = [min_x, max_y, min_z]
            vertices[7, :] = [max_x, max_y, min_z]
            vertices = vertices.dot(np.linalg.inv(rot))
            volume_bb = vol

        dim1 = np.linalg.norm(vertices[1]-vertices[0])
        dim2 = np.linalg.norm(vertices[2]-vertices[0])
        dim3 = np.linalg.norm(vertices[4]-vertices[0])
        if dim1 > dim2:
            bb_dim = [dim1, dim2, dim3]
        else:
            bb_dim = [dim2, dim1, dim3]

    return pca.inverse_transform(vertices), bb_dim
Graph Theory for Manufacturing

Optio was based on the theory that all manufacturing processes could be represented by a directed graph. However, the complexity of these graphs can often become pretty hairy. Some processes may have certain workstations where parts are separated but then come back together somewhere down the line. Some processes run parts in parallel, so the worker chooses which station to send the parts to next. Some processes even have cycles, where a part may need to visit the station multiple times. Some processes have all of these and more!

This algorithm was used to analyze a process and given a particular workstation that the parts are currently at, determine if the parts are ready to be sent on to the next station. That requires traversing the process graph to determine if the parts split, and if so, determine all the routes that parts must have come in from to be combined. It also requires ensuring that if the process has a cycle, that the part has reached station the desired number of times.


/**
 * Looks through the incoming routes and determines if all incoming parts are satisfied.
 * If the parts are satisfied, then the worker can move the lot along, otherwise they
 * need to wait until all parts come in. This gets complicated because you have to
 * backpropogate through the graph to determine if a part went through a diverging node
 * (i.e. choice or split node) at any point.
 *
 * Algorithm:
 * - Find process' start nodes
 * - Traverse the process graph until you reach a diverging node or the station ID that
 *   the parts are currently at. If the station ID is reached, exit.
 * - Recursively traverse all options out of the diverging node.
 *    The return value will be an 'expression' array whoes first value is && if the node
 *    was a split node and || if the node was a choice node. Subsequent values in the array
 *    will be the route IDs that the part must have / can have traversed.
 * - If you hit a converging node, you can remove duplicate routes in the expression. They
 *   have already been satisfied.
 *
 * @param {string} stationId The ID of the station that the parts are currently at.
 * @param {Process} process The process the parts are in
 * @param {array} routes The routes along the process
 * @param {object} stations All the process' station
 * @param {boolean} clean Whether or not to remove all null station IDs from the result
 * 
 * @returns {array} An expression that represents all the 'and' 'or' possibilities for a
 * part's path to have been satisfied. If all pieces of the expression are satisfied the
 * part is allowed to be released into the next station.
 * E.x. ['||' 'route1', ['&&', 'route2a', 'route2b']]
 */


export const handleMergeExpression = (stationId, process, routes, stations, clean=true) => {
  if (!process) return []

  const processRoutes = process.routes.map((routeId) => routes[routeId]);

  /***
   * Cleans up an expression.
   */
  const cleanExpression = (expression) => {
      if (!Array.isArray(expression)) {
          return expression
      } else if (expression[0] === null) {
          // If the first element is null, there is no && or ||. Its just the child ('through' node)
          return expression[1];
      } else {
          // Remove all nulls from the expression. Nulls are set when the recursion hits the end
          // of the process but didnt come across the desired stationId
          let nonNullExpression = expression.filter((el) => !!el);

          // If len < 2, this isnt even an expression (reached process end without hitting stationId)
          if (nonNullExpression.length < 2) {
              return null;
          } else if (
              // If all pieces are the same, theres no need for an && or ||
              nonNullExpression.every(
                  (element, idx) =>
                      idx === 0 ||
                      JSON.stringify(element) ===
                          JSON.stringify(nonNullExpression[1])
              )
          ) {
              return nonNullExpression[1];
          } else {
              // Otherwise, just return the expression
              return nonNullExpression;
          }
      }
  }

  let node, outgoingRoutes, nextRoutes, routeId;
  const recursiveExpand = (sExpression, traversed) => {
      let sExpressionCopy = deepCopy(sExpression);
      for (var entryIdx = 1; entryIdx < sExpression.length; entryIdx++) {
          routeId = sExpression[entryIdx];

          if (traversed.includes(routeId)) {
              sExpressionCopy[entryIdx] = null;
              continue
          }
          traversed.push(routeId)

          node = routes[routeId].unload;
          outgoingRoutes = getNodeOutgoing(node, processRoutes);
          if (node === stationId) { // We've found our station
              /**
               * This is where we handle cycles. When you find your station, THIS MAY NOT BE THE ONLY SOLUTION.
               * So instead, this is an || with all possible solutions. I.E. save this solution, but keep recursing in case a
               * route later down the graph has a solution as well.
               */

              if (outgoingRoutes.length === 0) {
                  sExpressionCopy[entryIdx] = routeId
              } else if (outgoingRoutes.length === 1) {
                  sExpressionCopy[entryIdx] = ['||', routeId, recursiveExpand([null, outgoingRoutes[0]._id], deepCopy(traversed))]
              } else {
                  nextRoutes = outgoingRoutes.map((route) => route._id);
                  if (
                      outgoingRoutes.some(
                          (route) => route.divergeType === "split"
                      )
                  ) {
                      sExpressionCopy[entryIdx] = ['||', routeId, recursiveExpand([
                          "&&",
                          ...nextRoutes,
                      ], deepCopy(traversed))]
                  } else {
                      sExpressionCopy[entryIdx] = ['||', routeId, recursiveExpand([
                          "||",
                          ...nextRoutes,
                      ], deepCopy(traversed))]
                  }
              }


              sExpressionCopy[entryIdx] = cleanExpression(sExpressionCopy[entryIdx]);
              
              
          } else if (outgoingRoutes.length === 0) {
              if (clean) {
                  sExpressionCopy[entryIdx] = null
              }
          } else if (outgoingRoutes.length === 1) {// Not a diverging node
              // NOTE, the recursive function only accepts an array, so we have to populate the first value with null.
              // This null is removed before the expression is returned \/
              sExpressionCopy[entryIdx] = recursiveExpand([
                  null,
                  outgoingRoutes[0]._id,
              ], deepCopy(traversed));
          } else { // Diverging node, new sub expression and expand
              nextRoutes = outgoingRoutes.map((route) => route._id);
              if (
                  outgoingRoutes.some(
                      (route) => route.divergeType === "split"
                  )
              ) {
                  sExpressionCopy[entryIdx] = recursiveExpand([
                      "&&",
                      ...nextRoutes,
                  ], deepCopy(traversed));
              } else {
                  sExpressionCopy[entryIdx] = recursiveExpand([
                      "||",
                      ...nextRoutes,
                  ], deepCopy(traversed));
              }
          }
      }

      return cleanExpression(sExpressionCopy)
  };

  const startNodes = findProcessStartNodes(processRoutes, stations); // Dont consider warehouses start nodes
  let startRouteExpression =
      process.startDivergeType === "split" ? ["&&"] : ["||"];
  for (var startNode of startNodes) {
      outgoingRoutes = getNodeOutgoing(startNode, processRoutes);
      if (outgoingRoutes.length === 1) {
          // NOTE, the recursive function only accepts an array, so we have to populate the first value with null.
          // This null is removed before the expression is returned \/
          startRouteExpression.push(
              recursiveExpand([null, outgoingRoutes[0]._id], [])
          );
      } else {
          nextRoutes = outgoingRoutes.map((route) => route._id);
          if (outgoingRoutes.some((route) => route.divergeType === "split")) {
              startRouteExpression.push(
                  recursiveExpand(["&&", ...nextRoutes], [])
              );
          } else {
              startRouteExpression.push(
                  recursiveExpand(["||", ...nextRoutes], [])
              );
          }
      }
  }

  const cleanedExpression = cleanExpression(startRouteExpression)
  return cleanedExpression;
};
Q* - A path planning algorithm for quants

Solomon is an algorithmic crypto trading bot that uses Proximal Policy Optimization the develop a policy for executing trades. A key insight of my implementation was a way to determine the maximal trajectory along a rollout to receive highest reward. This information could be used to augment the baseline of the training algorithm to improve performance. The maximal trajectory is non-trivial because of the fee associated with making a trade.

The insight I had was to envision the environment's state space as a directed graph, where each node represents a position (e.x. long, short, liquid) and each edge represented the reward associated with traversing from a position at timestamp t to a position at timestamp t + 1. I can then leverage a modern path planning algorithm like A* to determine a good approximation of the best trajectory for a given rollout length.


def calc_reward(position: Union[int, np.array], action: Union[int, np.array], delta: Union[float, np.array], spread: Union[float, np.array]) -> Union[float, np.array]:
	"""Calculate the reward for the previous action. The reward is the difference between
	the gain from the action the agent took and the gain if the agent had taken the other
	option. This reward function takes into account the actual gain and also the opportunity
	cost (or missed opportunity cost). This reward function is continuous and differentiable.

	Args:
		position (int): The position (short, out, long) that we were in at the previous timestep when the action took place.
		action (int): The action taken (-1: short, 0: out, 1: long)
		delta (float): The price change for the previous timestep to the current.
		spread (float): The bid-ask spread at the current timestep

	Returns:
		float: The reward scaled and normalized
	"""

	sign = (action * 2) - 1
	term1 = np.divide(1 + delta, 1 + spread) - 1
	term2 = np.add(delta, spread)

	T1 = np.multiply(1 - position, term1.reshape(-1, 1))
	T2 = np.multiply(position, term2.reshape(-1, 1))

	reward = np.multiply(sign, np.add(T1, T2))
	reward = np.clip(reward, -3, 3)

	if reward.size == 1:
		return reward[0][0]
	return reward


class Node:
	def __init__(self, prev_node, position, depth, g=0):
		self.prev_node = prev_node
		self.position = position
		self.depth = depth

		self.g = g

	def __lt__(self, other):
		# Necessary for collisions of same f value.
		# I flipped the sign because we use this in a maxheap.
		return self.g > other.g

def backtrace(node):
	path = []
	while node.prev_node is not None:
		path.append(node.position)
		node = node.prev_node
	path.append(node.position)
	return path[::-1]

def quant_star(position: bool, deltas: np.ndarray, spreads: np.ndarray, gamma: float) -> Tuple[float, List[bool]]:
	assert len(deltas) == len(spreads)
	stop_n = len(deltas)

	def heuristic(depth, position):
		if position:
			return np.sum(np.abs(deltas[depth:]))
		return 1

	start = Node(None, position, 0)
	frontier = [(0, start)]

	ret_so_far = {}
	ret_so_far[start] = 1

	while frontier:
		current = heapq.heappop(frontier)[1]

		if current.depth == stop_n - 1:
			break

		for action in [0, 1]:
			reward = calc_reward(current.position, action, deltas[current.depth], spreads[current.depth])
			child = Node(prev_node=current, position=action, depth=current.depth + 1)
			child.g = ret_so_far[current] + (gamma ** child.depth) * reward
			if child not in ret_so_far or child.g > ret_so_far[child]:
				ret_so_far[child] = child.g
				f = child.g + heuristic(child.depth, child.position)
				heapq.heappush(frontier, (-f, child))

	return current.g, backtrace(current)