progress
BIN
assets/img/writeup/docker/dockerfile_as_layers.png
Normal file
After Width: | Height: | Size: 94 KiB |
BIN
assets/img/writeup/git_intro/git_merging.png
Normal file
After Width: | Height: | Size: 24 KiB |
After Width: | Height: | Size: 44 KiB |
After Width: | Height: | Size: 66 KiB |
After Width: | Height: | Size: 22 KiB |
After Width: | Height: | Size: 51 KiB |
After Width: | Height: | Size: 52 KiB |
After Width: | Height: | Size: 34 KiB |
After Width: | Height: | Size: 66 KiB |
After Width: | Height: | Size: 61 KiB |
After Width: | Height: | Size: 142 KiB |
After Width: | Height: | Size: 74 KiB |
After Width: | Height: | Size: 48 KiB |
After Width: | Height: | Size: 65 KiB |
After Width: | Height: | Size: 32 KiB |
After Width: | Height: | Size: 33 KiB |
After Width: | Height: | Size: 262 KiB |
After Width: | Height: | Size: 254 KiB |
After Width: | Height: | Size: 320 KiB |
After Width: | Height: | Size: 350 KiB |
186
conf.py
|
@ -53,76 +53,146 @@ html_context = {"html_title": html_title}
|
||||||
|
|
||||||
pygments_style = "sas"
|
pygments_style = "sas"
|
||||||
|
|
||||||
drawio_headless = True
|
|
||||||
drawio_no_sandbox = True
|
|
||||||
|
|
||||||
|
|
||||||
class CanbusDbcLexer(RegexLexer):
|
class CanbusDbcLexer(RegexLexer):
|
||||||
name = "DBC"
|
name = "DBC"
|
||||||
|
|
||||||
tokens = {
|
tokens = {
|
||||||
'root' : [
|
"root": [
|
||||||
(r'[:\|\[\],\(\)]', token.Punctuation),
|
(r"[:\|\[\],\(\)]", token.Punctuation),
|
||||||
(r'[\@\-\+]', token.Operator),
|
(r"[\@\-\+]", token.Operator),
|
||||||
("CM_", token.Keyword, 'comment'),
|
("CM_", token.Keyword, "comment"),
|
||||||
("SG_", token.Keyword, "signal"),
|
("SG_", token.Keyword, "signal"),
|
||||||
("BO_", token.Keyword, 'msg'),
|
("BO_", token.Keyword, "msg"),
|
||||||
("BA_DEF_DEF_", token.Keyword, 'attrdefault'),
|
("BA_DEF_DEF_", token.Keyword, "attrdefault"),
|
||||||
("BA_DEF_", token.Keyword, 'attrdef'),
|
("BA_DEF_", token.Keyword, "attrdef"),
|
||||||
('BA_', token.Keyword, 'sigdefault')
|
("BA_", token.Keyword, "sigdefault"),
|
||||||
],
|
],
|
||||||
'msg': [
|
"msg": [
|
||||||
(r'(\s+)(\d+)(\s+)([\w_]+)(\s*?)(:)(\s+)(\d+)(\s+)(\w+)',
|
(
|
||||||
bygroups(token.Whitespace, token.Number, token.Whitespace, token.Name, token.Whitespace, token.Punctuation, token.Whitespace, token.Number, token.Whitespace, token.Name)),
|
r"(\s+)(\d+)(\s+)([\w_]+)(\s*?)(:)(\s+)(\d+)(\s+)(\w+)",
|
||||||
|
bygroups(
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Punctuation,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
),
|
||||||
|
),
|
||||||
],
|
],
|
||||||
'signal': [
|
"signal": [
|
||||||
(r'(\s+)([\w_]+)(\s+)(:)(\s+)(\d+)(\|)(\d+)(@)(\d+)([+-])(\s+)(\()([-\d\.]+)(,)([-\d\.]+)(\))(\s+)(\[)([\d-]+)(\|)([\d-]+)(\])(\s+)(".*?")(.*?\n)',
|
(
|
||||||
bygroups(token.Whitespace,
|
r'(\s+)([\w_]+)(\s+)(:)(\s+)(\d+)(\|)(\d+)(@)(\d+)([+-])(\s+)(\()([-\d\.]+)(,)([-\d\.]+)(\))(\s+)(\[)([\d-]+)(\|)([\d-]+)(\])(\s+)(".*?")(.*?\n)',
|
||||||
token.Name,
|
bygroups(
|
||||||
token.Whitespace,
|
token.Whitespace,
|
||||||
token.Punctuation,
|
token.Name,
|
||||||
token.Whitespace,
|
token.Whitespace,
|
||||||
token.Number,
|
token.Punctuation,
|
||||||
token.Punctuation,
|
token.Whitespace,
|
||||||
token.Number,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Number,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Whitespace,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Number,
|
token.Whitespace,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Number,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Whitespace,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Number,
|
token.Whitespace,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Number,
|
token.Number,
|
||||||
token.Punctuation,
|
token.Punctuation,
|
||||||
token.Whitespace,
|
token.Number,
|
||||||
token.String,
|
token.Punctuation,
|
||||||
token.Name,
|
token.Whitespace,
|
||||||
)),
|
token.String,
|
||||||
|
token.Name,
|
||||||
|
),
|
||||||
|
),
|
||||||
],
|
],
|
||||||
'comment': [
|
"comment": [
|
||||||
(r'(\s+)(BO_)(\s+)(\d+)(\s+)(".*?")(;)',
|
(
|
||||||
bygroups(token.Whitespace, token.Keyword, token.Whitespace, token.Number, token.Whitespace, token.String, token.Punctuation)),
|
r'(\s+)(BO_)(\s+)(\d+)(\s+)(".*?")(;)',
|
||||||
(r'(\s+)(SG_)(\s+)(\d+)(\s+)(\w+)(\s+)(".*?")(;)',
|
bygroups(
|
||||||
bygroups(token.Whitespace, token.Keyword, token.Whitespace, token.Number, token.Whitespace, token.Name, token.Whitespace, token.String, token.Punctuation))
|
token.Whitespace,
|
||||||
|
token.Keyword,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Whitespace,
|
||||||
|
token.String,
|
||||||
|
token.Punctuation,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
r'(\s+)(SG_)(\s+)(\d+)(\s+)(\w+)(\s+)(".*?")(;)',
|
||||||
|
bygroups(
|
||||||
|
token.Whitespace,
|
||||||
|
token.Keyword,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
token.Whitespace,
|
||||||
|
token.String,
|
||||||
|
token.Punctuation,
|
||||||
|
),
|
||||||
|
),
|
||||||
],
|
],
|
||||||
'attrdef': [
|
"attrdef": [
|
||||||
(r'(\s+)([\w_]+)(\s+)(".*?")(\s+)(\w+)(\s+)(.*?)(;)',
|
(
|
||||||
bygroups(token.Whitespace, token.Keyword, token.Whitespace, token.String, token.Whitespace, token.Keyword, token.Whitespace, token.Name, token.Punctuation))
|
r'(\s+)([\w_]+)(\s+)(".*?")(\s+)(\w+)(\s+)(.*?)(;)',
|
||||||
|
bygroups(
|
||||||
|
token.Whitespace,
|
||||||
|
token.Keyword,
|
||||||
|
token.Whitespace,
|
||||||
|
token.String,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Keyword,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
token.Punctuation,
|
||||||
|
),
|
||||||
|
)
|
||||||
],
|
],
|
||||||
'attrdefault': [
|
"attrdefault": [
|
||||||
(r'(\s+)(".*?")(\s+)(.*?)(;)',
|
(
|
||||||
bygroups(token.Whitespace, token.String, token.Whitespace, token.Name, token.Punctuation))
|
r'(\s+)(".*?")(\s+)(.*?)(;)',
|
||||||
|
bygroups(
|
||||||
|
token.Whitespace,
|
||||||
|
token.String,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
token.Punctuation,
|
||||||
|
),
|
||||||
|
)
|
||||||
],
|
],
|
||||||
'sigdefault': [
|
"sigdefault": [
|
||||||
(r'(\s+)(".*?")(\s+)(SG_)(\s+)(\d+)(\s+)(\S+)(\s+)([0-9\-\.]+)(;)',
|
(
|
||||||
bygroups(token.Whitespace, token.String, token.Whitespace, token.Keyword, token.Whitespace, token.Number, token.Whitespace, token.Name, token.Whitespace, token.Number, token.Punctuation))
|
r'(\s+)(".*?")(\s+)(SG_)(\s+)(\d+)(\s+)(\S+)(\s+)([0-9\-\.]+)(;)',
|
||||||
|
bygroups(
|
||||||
|
token.Whitespace,
|
||||||
|
token.String,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Keyword,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Name,
|
||||||
|
token.Whitespace,
|
||||||
|
token.Number,
|
||||||
|
token.Punctuation,
|
||||||
|
),
|
||||||
|
)
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
lexers['dbc'] = CanbusDbcLexer(startinline=True)
|
|
||||||
|
lexers["dbc"] = CanbusDbcLexer(startinline=True)
|
||||||
|
|
|
@ -7,7 +7,7 @@ Recent Posts
|
||||||
:titlesonly:
|
:titlesonly:
|
||||||
:hidden:
|
:hidden:
|
||||||
|
|
||||||
.. postlist:: 10
|
.. postlist::
|
||||||
:author: len0rd
|
:author: len0rd
|
||||||
:date: %Y-%m-%d
|
:date: %Y-%m-%d
|
||||||
:format: {date} - {title}
|
:format: {date} - {title}
|
||||||
|
|
178
posts/docker_intro.rst
Normal file
|
@ -0,0 +1,178 @@
|
||||||
|
Docker for SW Development
|
||||||
|
=========================
|
||||||
|
|
||||||
|
.. post:: 31, January 2025
|
||||||
|
:tags: development, infrastructure, tools, lunch-n-learn
|
||||||
|
:category: Projects
|
||||||
|
:author: len0rd
|
||||||
|
|
||||||
|
This is from a lunch-n-learn I hosted which aimed to showcase the features and advantages of using
|
||||||
|
docker during the development process. It includes with a few tutorials on how to use it.
|
||||||
|
|
||||||
|
Example code is hosted on my personal git server: https://git.lenord.me/len0rd/docker_demo
|
||||||
|
|
||||||
|
What is Docker
|
||||||
|
--------------
|
||||||
|
|
||||||
|
- Framework for defining and running low overhead sand-boxed environments called 'containers' on your host
|
||||||
|
|
||||||
|
- Its more than a "cheap VM"
|
||||||
|
|
||||||
|
.. image:: https://iot.samteck.net/wp-content/uploads/2018/10/vm-vs-docker.jpg
|
||||||
|
|
||||||
|
https://medium.com/@saschagrunert/demystifying-containers-part-i-kernel-space-2c53d6979504
|
||||||
|
|
||||||
|
|
||||||
|
.. image:: https://cdn-images-1.medium.com/max/1600/1*bIQOZL_pZujjrfaaYlQ_gQ.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Why docker?
|
||||||
|
-----------
|
||||||
|
|
||||||
|
- version-controlled binary dependency management
|
||||||
|
|
||||||
|
- documented and repeatable dependency installation in a living (version-controlled) file
|
||||||
|
|
||||||
|
- "containerize once and run everywhere"
|
||||||
|
|
||||||
|
- No more "it works on my machine"
|
||||||
|
|
||||||
|
- Industry-standard tool for CI/CD
|
||||||
|
|
||||||
|
Why docker for development?
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- consistent environment between developers and CI servers
|
||||||
|
|
||||||
|
- Project environment isolation - useful when working multiple projects
|
||||||
|
|
||||||
|
- Easy to experiment with and revert dependency changes - agile dependency management
|
||||||
|
|
||||||
|
Docker and Linux
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
- Docker is built atop core features of the Linux kernel (Namespaces) that are not available in Windows without WSL
|
||||||
|
|
||||||
|
- Docker desktop requires a license for commercial use. Docker server/CLI does not
|
||||||
|
|
||||||
|
docker image
|
||||||
|
------------
|
||||||
|
|
||||||
|
- Docker image is a snapshot of a linux rootfs
|
||||||
|
|
||||||
|
- Images can be built manually or downloaded from a registry
|
||||||
|
|
||||||
|
Command: ``docker image [--help]``
|
||||||
|
|
||||||
|
docker registry
|
||||||
|
---------------
|
||||||
|
|
||||||
|
- Pre-built images published online you can download
|
||||||
|
|
||||||
|
- Anyone can host a registry. Largest: https://hub.docker.com/search?q=
|
||||||
|
|
||||||
|
Command: ``docker pull <image_name>``
|
||||||
|
|
||||||
|
.. admonition:: Example
|
||||||
|
|
||||||
|
Take a look at `example 1 <https://git.lenord.me/len0rd/docker_demo/src/branch/develop/01_registry_pull>`_
|
||||||
|
|
||||||
|
docker container
|
||||||
|
----------------
|
||||||
|
|
||||||
|
- Container is a spawned docker image
|
||||||
|
|
||||||
|
- Container *images* become *containers* at runtime
|
||||||
|
|
||||||
|
- You can spawn many containers from a single image
|
||||||
|
|
||||||
|
- Containers can run one or many processes, can provide user shells
|
||||||
|
|
||||||
|
Managing existing containers: ``docker container [--help]``
|
||||||
|
|
||||||
|
dockerfiles
|
||||||
|
-----------
|
||||||
|
|
||||||
|
- How you define custom images
|
||||||
|
|
||||||
|
- Good documentation: https://docs.docker.com/engine/reference/builder/
|
||||||
|
|
||||||
|
- Dockerfiles build off images. Each step in a dockerfile is a snapshot of changes to a linux rootfs
|
||||||
|
|
||||||
|
Command: ``docker build --help``
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/docker/dockerfile_as_layers.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
.. admonition:: Example
|
||||||
|
|
||||||
|
Take a look at `example 2 <https://git.lenord.me/len0rd/docker_demo/src/branch/develop/02_simple_dockerfile>`_
|
||||||
|
|
||||||
|
Running Containers
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- when a container is run you are in full control of what parts have access to the outside world
|
||||||
|
|
||||||
|
- ports forwarded from container to host
|
||||||
|
|
||||||
|
- network configuration in general
|
||||||
|
|
||||||
|
- mounting files/folders from the host within the container
|
||||||
|
|
||||||
|
Running: ``docker run --help``
|
||||||
|
|
||||||
|
docker cache
|
||||||
|
------------
|
||||||
|
|
||||||
|
- Every step in a dockerfile is cached. if a step in the middle of the file changes, the image is rebuilt from that point onward
|
||||||
|
|
||||||
|
- https://docs.docker.com/build/cache/
|
||||||
|
|
||||||
|
cache minimization
|
||||||
|
^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- docker caches can take up a lot of space
|
||||||
|
|
||||||
|
- Best practice is to combine logically linked commands into single steps to reduce the amount of cache a dockerfile build produces
|
||||||
|
|
||||||
|
- clearing out old cache: ``docker builder prune``
|
||||||
|
|
||||||
|
|
||||||
|
multi-stage builds
|
||||||
|
------------------
|
||||||
|
|
||||||
|
- newish (in the last couple years) feature, now standard is docker's BuildKit backend
|
||||||
|
|
||||||
|
- allows definition of distinct stages in the dockerfile to allow parallel build and reduce unnecessary dependencies
|
||||||
|
|
||||||
|
- https://docs.docker.com/build/buildkit/
|
||||||
|
|
||||||
|
.. admonition:: Example
|
||||||
|
|
||||||
|
Take a look at `example 4 <https://git.lenord.me/len0rd/docker_demo/src/branch/develop/04_multistage_docker_file>`_
|
||||||
|
|
||||||
|
.. image:: https://git.lenord.me/len0rd/docker_demo/raw/branch/develop/04_multistage_docker_file/stage_graph.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
DevContainers: an additional layer on top of docker
|
||||||
|
---------------------------------------------------
|
||||||
|
|
||||||
|
- Feature first added by VSCode, eventually published as an open standard by Microsoft:
|
||||||
|
|
||||||
|
- https://containers.dev/
|
||||||
|
|
||||||
|
- Now multiple IDEs (IntelliJ, Visual Studio, NeoVim) have support for the standard
|
||||||
|
|
||||||
|
.. image:: https://code.visualstudio.com/assets/docs/devcontainers/containers/architecture-containers.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
docker-compose: container orchestration
|
||||||
|
---------------------------------------
|
||||||
|
|
||||||
|
- ``docker run`` and ``docker build`` commands can get long and tedious for more complex containers
|
||||||
|
|
||||||
|
- ``docker compose`` solves this. Sits on top of docker and supports building/running multiple containers via yaml
|
||||||
|
|
||||||
|
.. admonition:: Example
|
||||||
|
|
||||||
|
Take a look at `example 5 <https://git.lenord.me/len0rd/docker_demo/src/branch/develop/05_complex_run_with_compose>`_
|
517
posts/git_intro.rst
Normal file
|
@ -0,0 +1,517 @@
|
||||||
|
.. include this for helpful ascii symbols like degrees. See https://www.faschingbauer.me/about/site/recipes/isonum.html
|
||||||
|
.. include:: <isonum.txt>
|
||||||
|
|
||||||
|
Git basics
|
||||||
|
==========
|
||||||
|
|
||||||
|
.. post:: 16, December 2024
|
||||||
|
:tags: embedded, development, infrastructure, tools, lunch-n-learn
|
||||||
|
:category: Projects
|
||||||
|
:author: len0rd
|
||||||
|
|
||||||
|
This is from a lunch-n-learn I hosted. Yet another intro to git is not necessary on the internet, but I wanted to store
|
||||||
|
mine someplace easily accessible for when I need it again. This introduction is geared towards
|
||||||
|
Software Engineers and touches on some of the inner workings/concepts git employs.
|
||||||
|
|
||||||
|
Intro
|
||||||
|
-----
|
||||||
|
|
||||||
|
- *Distributed* Version Control System (VCS)
|
||||||
|
|
||||||
|
- The dominant source code management tool
|
||||||
|
|
||||||
|
- Git != Github
|
||||||
|
|
||||||
|
- Git is open source
|
||||||
|
|
||||||
|
- Github is an implementation of a git server. It hosts git repositories and a bunch of extra features on top
|
||||||
|
|
||||||
|
- git has a higher learning curve (than other VCS'), but in exchange gives you a lot more control
|
||||||
|
|
||||||
|
- There are many gui applications for git, but its valuable to understand what these apps are doing (Command Line Interface)
|
||||||
|
|
||||||
|
Distributed?
|
||||||
|
------------
|
||||||
|
|
||||||
|
Centralized (ie: Subversion/SVN) vs distributed (Git)
|
||||||
|
|
||||||
|
.. image:: https://k21academy.com/wp-content/uploads/2020/07/dvcs.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
History
|
||||||
|
-------
|
||||||
|
|
||||||
|
- Originally written by Linus Torvalds, creator of linux. Starting in 2005
|
||||||
|
|
||||||
|
- built to address the problems with Version Control Systems that affected linux development the most
|
||||||
|
|
||||||
|
- fast merging
|
||||||
|
|
||||||
|
- strong safeguards against corruption
|
||||||
|
|
||||||
|
- distributed workflow
|
||||||
|
|
||||||
|
- Git describes itself as "the stupid content tracker"
|
||||||
|
|
||||||
|
Getting started
|
||||||
|
---------------
|
||||||
|
|
||||||
|
- Create a new repo
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
git init
|
||||||
|
|
||||||
|
- Download an existing repo for the first time:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
git clone
|
||||||
|
|
||||||
|
The 3 States of a file
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
.. image:: https://jwiegley.github.io/git-from-the-bottom-up/images/lifecycle.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
|
||||||
|
**Viewing what stage things are in:**
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
git status
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
``git status`` is one of gits most useful commands! It provides a lot of information on the current state of your repository
|
||||||
|
|
||||||
|
Working tree / working directory
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
Where all changes live at first. The directory you see on your machine.
|
||||||
|
2 types of files in the working directory:
|
||||||
|
|
||||||
|
- **Tracked files**: files git knows about. These are files that were in a prior commit
|
||||||
|
- **Untracked files**: any files in the working directory that have never been committed and are not staged
|
||||||
|
|
||||||
|
Staging area
|
||||||
|
^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Also called the index
|
||||||
|
- a spot to register changes from the working directory in preparation for a commit. a middle ground thats unique to git
|
||||||
|
- The version of a file in the staging area is separate from the same file in your working directory
|
||||||
|
|
||||||
|
Taking a file from Working Dir |rarr| staging: ``git add <file>``
|
||||||
|
|
||||||
|
Easiest to do in a visual editor when the your making a more complex index/commit (Example: VsCode)
|
||||||
|
|
||||||
|
Commits in the repository
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Commits are snapshots of the entire repository
|
||||||
|
|
||||||
|
- changes in the staging area turn into a commit with ``git commit``
|
||||||
|
|
||||||
|
- shortcut: ``git commit -m "commit message..."`` if you dont need a full editor for the commit message
|
||||||
|
|
||||||
|
- commits build off one another to create the repository's "history"
|
||||||
|
|
||||||
|
- Helpful way to view history: ``git log``
|
||||||
|
|
||||||
|
- Commits as a visual graph: ``git log --oneline --graph``
|
||||||
|
|
||||||
|
How commits work
|
||||||
|
----------------
|
||||||
|
|
||||||
|
- Under the hood git has 3 fundamental objects:
|
||||||
|
|
||||||
|
- **blob**: "Binary Large Object". Just the contents of a file
|
||||||
|
|
||||||
|
- **tree**: a directory listing may contain blobs or more trees (imagine ``ls`` as an object)
|
||||||
|
|
||||||
|
- **commit**: snapshot of the working directory
|
||||||
|
|
||||||
|
- (There are also *tag* and *packfile* objects, but not important for this discussion)
|
||||||
|
|
||||||
|
.. image:: https://www.freecodecamp.org/news/content/images/2020/12/image-37.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
As a directory:
|
||||||
|
|
||||||
|
.. code-block::
|
||||||
|
|
||||||
|
DOCS/
|
||||||
|
PIC.PNG
|
||||||
|
1.TXT
|
||||||
|
TEST.JS
|
||||||
|
|
||||||
|
Adding another commit
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
.. image:: https://www.freecodecamp.org/news/content/images/2020/12/image-41.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
- Each commit is a full-snapshot of the repository, not just a diff. Changed BLOBS are added to the commit, Unchanged blobs/trees reference prior commits
|
||||||
|
|
||||||
|
.. admonition:: Aside (not important for typical git usage)
|
||||||
|
|
||||||
|
You can prove the above model to yourself by using ``git cat-file`` within a repo
|
||||||
|
|
||||||
|
- Start with ``git cat-file -p <COMMIT_SHA>``
|
||||||
|
|
||||||
|
- commits point to a ``tree: <SHA>``
|
||||||
|
|
||||||
|
- ``git cat-file -p <TREE_SHA>`` can display what a tree object looks like
|
||||||
|
|
||||||
|
- from there you can get more tree or blob SHAs
|
||||||
|
|
||||||
|
- ``cat-file -p <BLOB_SHA>`` just prints out the file itself
|
||||||
|
|
||||||
|
SHAs
|
||||||
|
^^^^
|
||||||
|
|
||||||
|
- SHA = Secure Hashing Algorithm
|
||||||
|
|
||||||
|
- 40-character hexadecimal SHA256 is used to to identify every object in a git repository quickly
|
||||||
|
|
||||||
|
- these are the little tags of random numbers/letters in the above images (ie: ``62E7A``)
|
||||||
|
|
||||||
|
- (in the images they are abbreviated from 40 characters for simplicity)
|
||||||
|
|
||||||
|
- Concept of hashing algorithms:
|
||||||
|
|
||||||
|
- for any amount of input data, generate a unique fixed-length *hash* (code of 40 characters used by git)
|
||||||
|
|
||||||
|
- repeatable. when fed the same data, you get the same hash
|
||||||
|
|
||||||
|
- unique: 2 different inputs should not generate the same hash
|
||||||
|
|
||||||
|
- if your input data changes by only 1 bit, the resulting hash will be different
|
||||||
|
|
||||||
|
- See https://en.wikipedia.org/wiki/Cryptographic_hash_function
|
||||||
|
|
||||||
|
|
||||||
|
Remotes
|
||||||
|
-------
|
||||||
|
|
||||||
|
- Remotes are gits way of pointing to non-local repository locations (like bitbucket or github for instance)
|
||||||
|
- When your local changes are to be on the git server use `git push`
|
||||||
|
- When you want to get everyone's latest changes from the server: `git pull`
|
||||||
|
- this `fetch` es changes from the remote repository and merges them into your local copy
|
||||||
|
- you can get remote changes without merging them into your local copy with `git fetch`
|
||||||
|
|
||||||
|
Managing remotes
|
||||||
|
^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
In your local copy of a repository, you have control over what remotes your code is sent to/retrieved from
|
||||||
|
|
||||||
|
- The default remote is ``origin``
|
||||||
|
|
||||||
|
- When first setting up a repository with ``git clone`` , ``origin`` will be set to the URL you're cloning from
|
||||||
|
|
||||||
|
- list configured remotes ``git remote -v``
|
||||||
|
|
||||||
|
- Add a new remote: ``git remote add <name> <git_url>``
|
||||||
|
|
||||||
|
- Update the URL of a remote: ``git remote set-url <name> <new_git_url>``
|
||||||
|
|
||||||
|
- All push/pull commands can be directed at specific remotes
|
||||||
|
|
||||||
|
- ex: ``git push origin``: push my local code on my current branch to ``origin``
|
||||||
|
|
||||||
|
- Another common remote name: ``upstream``
|
||||||
|
|
||||||
|
- Used when 'forking' another repository on github
|
||||||
|
|
||||||
|
- ex: https://github.com/len0rd/pyftdi
|
||||||
|
|
||||||
|
Branching
|
||||||
|
---------
|
||||||
|
|
||||||
|
- fundamental to collaborative work in a repository
|
||||||
|
|
||||||
|
- Branches are just nice pointers to commit SHAs
|
||||||
|
|
||||||
|
- Under the hood: branches live in ``.git/refs/``. They're just files with a the commit SHA they currently point to
|
||||||
|
|
||||||
|
.. image:: https://www.freecodecamp.org/news/content/images/2020/12/image-42.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Branch Commands
|
||||||
|
^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Create new branch ``git branch``
|
||||||
|
|
||||||
|
- select branch for working directory: ``git checkout``
|
||||||
|
|
||||||
|
- shortcut: ``git checkout -b <new_branch_name>``
|
||||||
|
|
||||||
|
- what branch am i on? ``git status``
|
||||||
|
|
||||||
|
- ``HEAD``: special git pointer that points to the branch currently checked out in your local copy
|
||||||
|
|
||||||
|
.. image:: https://www.freecodecamp.org/news/content/images/2020/12/image-43.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Adding a commit to a branch. Branches automatically update to the latest commit applied to them
|
||||||
|
|
||||||
|
Merging
|
||||||
|
-------
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/git_intro/git_merging.png
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Most common merge method: 3-way merge
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- pictured above
|
||||||
|
|
||||||
|
- The three commits used in this merge:
|
||||||
|
|
||||||
|
- Head of destination
|
||||||
|
|
||||||
|
- Head of branch to be merged
|
||||||
|
|
||||||
|
- the common ancestor (allows git to know the exact diffs its combining)
|
||||||
|
|
||||||
|
- The merge operation itself is usually facilitated by an online tool, but you have to address merge conflicts manually
|
||||||
|
|
||||||
|
- Start a merge: ``git merge <SOURCE_BRANCH>``
|
||||||
|
|
||||||
|
- Note! All merge commands merge into the branch you are currently on
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
git checkout develop
|
||||||
|
# this will merge feature/a into develop
|
||||||
|
git merge feature/a
|
||||||
|
|
||||||
|
- There are many other merge types/strategies that will be discussed later
|
||||||
|
|
||||||
|
Dealing with conflicts
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Conflicts can occur for a few reasons (when merging A and B):
|
||||||
|
|
||||||
|
- file was deleted on A but edited on B
|
||||||
|
|
||||||
|
- same portion of a file was edited on A and B
|
||||||
|
|
||||||
|
- when there's a conflict, git will halt the merge, mark the conflicts, and tell you to resolve them
|
||||||
|
|
||||||
|
- Conflict resolution too hard? you can always back out: ``git merge --abort``
|
||||||
|
|
||||||
|
- What a conflict looks like
|
||||||
|
|
||||||
|
.. code-block::
|
||||||
|
|
||||||
|
hello
|
||||||
|
<<<<<<< HEAD
|
||||||
|
there! i want to add stuff here
|
||||||
|
=======
|
||||||
|
there.
|
||||||
|
another
|
||||||
|
line!
|
||||||
|
>>>>>>> feature/tmiller/more_stuff
|
||||||
|
|
||||||
|
- To resolve you need to decide if you want the current branches version ``HEAD`` / "ours", the branch being merged version (``<BRANCH_NAME>`` / "theirs") or some combination of both.
|
||||||
|
|
||||||
|
- Regardless, you should always remove the marker lines (``<<<<<``, ``>>>>``, etc) before finishing the merge
|
||||||
|
|
||||||
|
- GUIs/Editors can be helpful here (VSCode)
|
||||||
|
|
||||||
|
- To finish the merge:
|
||||||
|
|
||||||
|
- add conflicted (but now resolved) files with ``git add``
|
||||||
|
|
||||||
|
- commit with ``git commit`` (git will autofill the message)
|
||||||
|
|
||||||
|
Stashing
|
||||||
|
--------
|
||||||
|
|
||||||
|
- Temporarily save working-tree changes so you can come back to them later
|
||||||
|
|
||||||
|
- Useful when you need to switch branches but your current work isnt ready to commit yet
|
||||||
|
|
||||||
|
- (Mercurial has a plugin called ``shelve`` that works in a similar way)
|
||||||
|
|
||||||
|
- Stashes are stored in a stack-like structure
|
||||||
|
|
||||||
|
- stack-like because it works like a stack by default, but you can operate on things in the middle of the stack without issue
|
||||||
|
|
||||||
|
Working with stashes
|
||||||
|
^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
`Git stash options <https://wac-cdn.atlassian.com/dam/jcr:d6fec41a-dc66-4af6-8b0f-c23d271eaf8e/01.svg?cdnVersion=1352>`_
|
||||||
|
|
||||||
|
- Current stashes: ``git stash list``
|
||||||
|
|
||||||
|
- See what's in a stash: ``git stash show [-p] [STASH_NAME]``
|
||||||
|
|
||||||
|
- Apply a stash: ``git stash apply [STASH_NAME]``
|
||||||
|
|
||||||
|
- Drop a stash from the stack: ``git stash drop [STASH_NAME]``
|
||||||
|
|
||||||
|
- Apply the latest stash and drop from the stack: ``git stash pop``
|
||||||
|
|
||||||
|
How stashes work
|
||||||
|
^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
Internally each stash is just a set of commit objects stored in the ``.git/`` folder
|
||||||
|
|
||||||
|
Aliases - make git more convenient
|
||||||
|
----------------------------------
|
||||||
|
|
||||||
|
I use bash aliases to make using git from a terminal faster/easier. These can be added to your ``~/.bashrc`` for more convenient git usage. Here are some that you may find useful.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
alias gs='git status'
|
||||||
|
alias ga='git add'
|
||||||
|
alias gc='git commit'
|
||||||
|
alias gpush='git push'
|
||||||
|
alias gpull='git pull'
|
||||||
|
# update git submodules to the correct commit
|
||||||
|
alias gsub='git submodule update --init --recursive'
|
||||||
|
# prune away local copies of branches that have been merged on remote
|
||||||
|
alias gprune='git fetch -p && git branch --merged | grep -i -v -E "master|develop|main|dev|development" | xargs git branch -d'
|
||||||
|
alias gdiff='git diff'
|
||||||
|
# git log with one line per commit and a nice graph
|
||||||
|
alias glog="git log --oneline --decorate --graph --branches --remotes --tags"
|
||||||
|
|
||||||
|
|
||||||
|
Git's also has an `alias system <https://git-scm.com/book/en/v2/Git-Basics-Git-Aliases>`_. Aliases are stored in Git's config file (``~/.gitconfig``). You can use these to perform more complex git commands. All aliases here will start with ``git``. Example:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
[alias]
|
||||||
|
# list most recently visited branches see http://ses4j.github.io/2020/04/01/git-alias-recent-branches/
|
||||||
|
recent = !git reflog show --pretty=format:'%gs ~ %gd' --date=relative | grep 'checkout:' | grep -oE '[^ ]+ ~ .*' | awk -F~ '!seen[$1]++' | head -n 10 | awk -F' ~ HEAD@{' '{printf(\" \\033[33m%s: \\033[37m %s\\033[0m\\n\", substr($2, 1, length($2)-1), $1)}'
|
||||||
|
|
||||||
|
Usage: ``git recent``
|
||||||
|
|
||||||
|
At the end of the day, most of the git commands you use are just aliases around a set of more fundamental commands
|
||||||
|
|
||||||
|
Git Flow
|
||||||
|
--------
|
||||||
|
|
||||||
|
- "git flow" is the term used to describe accepted best practices for git usage on collaborative projects.
|
||||||
|
|
||||||
|
- Everyone using git typically follows some or all of git flow
|
||||||
|
|
||||||
|
- The highlights:
|
||||||
|
|
||||||
|
- Primary work branch, typically ``develop``, where all new features get merged
|
||||||
|
|
||||||
|
- committing directly to the ``develop`` branch is not allowed
|
||||||
|
|
||||||
|
- Add features or fix bugs in separate feature / bugfix branches. Then merge them into ``develop``
|
||||||
|
|
||||||
|
- For releases, you should branch off ``develop`` and tag the commit used for a specific release
|
||||||
|
|
||||||
|
- Release branches should only receive bugfixes, if anything
|
||||||
|
|
||||||
|
- ``release`` branch is merged back into ``develop``
|
||||||
|
|
||||||
|
- Reality: sometimes a release will have hacky/temporary fixes to get a product out the door. So the merge back into ``develop`` will contain a portion of the ``release/`` branch changes, not all
|
||||||
|
|
||||||
|
.. image:: https://wac-cdn.atlassian.com/dam/jcr:cc0b526e-adb7-4d45-874e-9bcea9898b4a/04%20Hotfix%20branches.svg?cdnVersion=2536
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Other recommendations
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
#. Avoid copy-pasting third-party code into a project
|
||||||
|
|
||||||
|
- Its a pain to track and update code copied in. it adds a bunch of unnecessary stuff to your history and can be difficult to update
|
||||||
|
|
||||||
|
- Alternatives:
|
||||||
|
|
||||||
|
- use a package manager built for the source code language
|
||||||
|
|
||||||
|
- Fork + Submodule strategy (discussed later)
|
||||||
|
|
||||||
|
- guarantees the code is always internally accessible
|
||||||
|
|
||||||
|
- easy to track internal changes to the code
|
||||||
|
|
||||||
|
- easy to pull in upstream fixes/updates
|
||||||
|
|
||||||
|
- easy to push any approved internal changes upstream
|
||||||
|
|
||||||
|
#. Delete branches once they are merged
|
||||||
|
|
||||||
|
- Unless there's a strong reason not to (like if its a ``release/`` branch)
|
||||||
|
|
||||||
|
#. Most git hosts (github, bitbucket) allow you to setup rules to enforce git-flow. Use them.
|
||||||
|
|
||||||
|
- restrict commits on ``develop`` / ``release/`` branches to merge commits only
|
||||||
|
|
||||||
|
- Require approval to merge
|
||||||
|
|
||||||
|
- Require build/tests to pass to merge
|
||||||
|
|
||||||
|
Terminology Reference
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
When searching online, its helpful to use the proper terms to describe your problem
|
||||||
|
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| term | explanation |
|
||||||
|
+===================+=======================================================================================+
|
||||||
|
| Repository / repo | a project tracked by git. a folder that contains a ``.git/`` folder |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| Index | aka "Staging". Area for files/diffs before they are committed |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| Working directory | Your local copy of the repository where you make changes |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| Commits | Snapshot of repository state |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| History | The commits, branches, merges etc in a repository make that repos history |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
| SHA | Secure Hashing Algorithm. A unique 40 character code used to identify all git objects |
|
||||||
|
+-------------------+---------------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
Other Useful Commands
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| Description | Command(s) |
|
||||||
|
+=====================================================================+=====================================================================+
|
||||||
|
| **List commits between refs** | ``git log <since>...<until>`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Stash a specific file** | ``git stash push -m <stashname> <file>`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Create a patch** | |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| of current diff | ``git diff > <patch_file_name>`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| of commits (range) | ``git format-patch <since_commit>`` ex: |
|
||||||
|
| | ``HEAD~2`` = last 2 commits |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Apply patches** | ``git apply <patch_file>`` *Can be glob* |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Pull rebased branch from remote** (also reset an amended commit) | ``git reset --hard origin/<branch/name>`` |
|
||||||
|
| Will DELETE local working tree changes! | |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Put last commit back into staging** | ``git reset --soft HEAD^`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Update Remote URL** | ``get remote set-url <REMOTE> <NEW_URL>`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Delete a local branch** | ``git branch -D <BRANCH_NAME>`` |
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
| **Graph of latest commits** | ``git log --oneline --decorate --graph --branches --remotes --tags``|
|
||||||
|
+---------------------------------------------------------------------+---------------------------------------------------------------------+
|
||||||
|
|
||||||
|
References
|
||||||
|
----------
|
||||||
|
|
||||||
|
- https://en.wikipedia.org/wiki/Git
|
||||||
|
- https://git-scm.com/docs
|
||||||
|
- https://www.freecodecamp.org/news/git-internals-objects-branches-create-repo/
|
||||||
|
- https://jwiegley.github.io/git-from-the-bottom-up/
|
||||||
|
- https://www.atlassian.com/git/tutorials/saving-changes/git-stash
|
||||||
|
- https://www.atlassian.com/git/tutorials/using-branches/git-merge
|
||||||
|
- https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-using-the-command-line
|
||||||
|
- https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow
|
|
@ -1,13 +1,18 @@
|
||||||
.. myWebsite:
|
.. myWebsite:
|
||||||
|
|
||||||
My Website
|
My Website [Old]
|
||||||
==========
|
================
|
||||||
|
|
||||||
.. post:: 31, July 2018
|
.. post:: 31, July 2018
|
||||||
:tags: coding, diy, old
|
:tags: coding, diy, old
|
||||||
:category: Projects
|
:category: Projects
|
||||||
:author: len0rd
|
:author: len0rd
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
This page describes how I initially implemented this website back in 2018. I've since moved to a much
|
||||||
|
simpler solution using `Sphinx Docs <https://www.sphinx-doc.org/en/master/>`_ and `ablog <https://ablog.readthedocs.io/en/stable/>`_
|
||||||
|
|
||||||
Starting out with this website, I had essentially no knowledge of modern web technologies. I knew that I wanted something modern but also easy to maintain that I could use well into the future.
|
Starting out with this website, I had essentially no knowledge of modern web technologies. I knew that I wanted something modern but also easy to maintain that I could use well into the future.
|
||||||
|
|
||||||
The end result is the site you see here. By no means perfect or beautiful, but functional and a place where I can store guides mainly for my benefit. But maybe for your benefit too? I certainly dont know who's reading this ¯\\\_(ツ)_/¯
|
The end result is the site you see here. By no means perfect or beautiful, but functional and a place where I can store guides mainly for my benefit. But maybe for your benefit too? I certainly dont know who's reading this ¯\\\_(ツ)_/¯
|
||||||
|
@ -32,12 +37,8 @@ Express
|
||||||
|
|
||||||
Originally I started this site as a pure html/bootstrap affair. This worked for all of 2 days until I got sick of copying and pasting code all over the place. While I had no desire to maintain duplicate copies of code, I was even less interested in using some massive overkill framework (as an embedded dev, I have a need for speed). Low and behold: ExpressJS! The perfect minimal framework solution for my problem. Express has a concept of 'pages' and 'partials'. A page defines the overall structure of a static webpage (say my home page). Partials define chunks/components of that page that are shared in other locations. So for example, all the html for my navigation/ header bar has its own partial, as does the footer. Then in a page, to use this content you can simply add a ``<% include`` as if you were writing a C program! Express was speaking my language.
|
Originally I started this site as a pure html/bootstrap affair. This worked for all of 2 days until I got sick of copying and pasting code all over the place. While I had no desire to maintain duplicate copies of code, I was even less interested in using some massive overkill framework (as an embedded dev, I have a need for speed). Low and behold: ExpressJS! The perfect minimal framework solution for my problem. Express has a concept of 'pages' and 'partials'. A page defines the overall structure of a static webpage (say my home page). Partials define chunks/components of that page that are shared in other locations. So for example, all the html for my navigation/ header bar has its own partial, as does the footer. Then in a page, to use this content you can simply add a ``<% include`` as if you were writing a C program! Express was speaking my language.
|
||||||
|
|
||||||
Static Project Pages [old]
|
Static Project Pages
|
||||||
--------------------------
|
--------------------
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
I've replaced this implementation with Sphinx docs
|
|
||||||
|
|
||||||
The bulk of the effort for me was sunk into generating the project writeup pages (like the page you're reading this off of right now!). I wanted them to be simple static text, images and video. But I didn't want the complexity of using a whole framework like wordpress, and I definitely wasn't into the idea of writting everything in html. I wanted my writeups to be in a portable format I could easily migrate or use in other places in the future.
|
The bulk of the effort for me was sunk into generating the project writeup pages (like the page you're reading this off of right now!). I wanted them to be simple static text, images and video. But I didn't want the complexity of using a whole framework like wordpress, and I definitely wasn't into the idea of writting everything in html. I wanted my writeups to be in a portable format I could easily migrate or use in other places in the future.
|
||||||
|
|
||||||
|
|
93
posts/unified_uas_messaging.rst
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
Unified Messaging Solution for Small UAS
|
||||||
|
========================================
|
||||||
|
|
||||||
|
.. post:: 16, September 2023
|
||||||
|
:tags: embedded, development, sw-architecture
|
||||||
|
:category: Projects
|
||||||
|
:author: len0rd
|
||||||
|
|
||||||
|
A major challenge I've repeatedly encountered while working in embedded software is
|
||||||
|
communication protocols in a heterogeneous and evolving system. The main difficulty
|
||||||
|
lies in obtaining modern messaging system conveniences while supporting the lowest-common
|
||||||
|
denominator in your system (ie: a tiny Cortex-M0 running in the system).
|
||||||
|
|
||||||
|
To clarify:
|
||||||
|
|
||||||
|
- Heterogeneous = System made up of multiple MCUs/SOMs all with different levels of capability
|
||||||
|
- Evolving = System who's architecture (number of MCUs, their location), is actively being changed
|
||||||
|
|
||||||
|
This post is a high-level overview of a solution I previously developed to address some of these issues in a small Unmanned Aircraft System (UAS) project. It describes the motivation behind the solution, the solution itself, and its short-comings/lessons-learned.
|
||||||
|
|
||||||
|
Motivation
|
||||||
|
----------
|
||||||
|
|
||||||
|
Small UAS Comm Setups
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
Here's a typical UAS setup you may see in a hobby-ist scenario. Its using an open-source autopilot (PX4) and a telemetry radio to commincate with a ground station.
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/unified_messaging_small_uas/small_uas_setup_a.png
|
||||||
|
:width: 400
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Here's another option that's a little more robust. It includes an open-source autopilot along with a "companion
|
||||||
|
computer", typically some kind of ARM SOM, for performing more CPU-intensive but less time-critical tasks.
|
||||||
|
|
||||||
|
In this scenario you'll often see a more embedded-friendly protocol used for communication between the autopilot
|
||||||
|
and companion computer. While a more robust and feature-rich network-stack-based protocol is used for communication with the companion computer and ground station (ROS, protobuf, proprietary)
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/unified_messaging_small_uas/small_uas_setup_b.png
|
||||||
|
:width: 400
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
Small UAS Problem Architectures
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
The above UAS architectures are common and well-supported in the open-source/hobbyist UAS community. However problems begin to arise with more complex requirements. Such as:
|
||||||
|
|
||||||
|
- UAS must support communication with a smart battery
|
||||||
|
|
||||||
|
- UAS must support optional communication with a hot-swapable gimbal
|
||||||
|
|
||||||
|
- UAS must support ground station communication with or without a companion computer
|
||||||
|
|
||||||
|
- UAS software should be robust to network topology evolution as the system design is refined.
|
||||||
|
|
||||||
|
Here's what a couple of these scenarios may look like:
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/unified_messaging_small_uas/small_uas_setup_c.png
|
||||||
|
:width: 350
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/unified_messaging_small_uas/small_uas_setup_d.png
|
||||||
|
:width: 350
|
||||||
|
:class: dark-light
|
||||||
|
|
||||||
|
|
||||||
|
System Challenges
|
||||||
|
^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Heterogeneous system. Running anything from a baremetal Cortex-M0 to a Linux SOM. Vastly different capabilities
|
||||||
|
|
||||||
|
- Network topology is constantly evolving as the system is refined. Some chips/boards are consolidated, others are broken out, while others are changed to be optional modules in the final product
|
||||||
|
|
||||||
|
Software Challenges
|
||||||
|
^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
- Maintainability of separate messaging frameworks
|
||||||
|
- Maintaining even more than one messaging framework in a project quickly becomes tedious. From past experience I knew that this scenario can make projects difficult to understand and work in. Maintainable code should be the highest priority when architecting software solutions
|
||||||
|
|
||||||
|
- Efficiency of separate messaging frameworks
|
||||||
|
- If each device uses a separate protocol to communicate with its neighbors, cycles will be lost translating messages from one protocol to the other. If any of these protocols have additional logic (routing, messaging state), it will multiply the software difficulty and overhead
|
||||||
|
|
||||||
|
- Convenient data sharing between modules
|
||||||
|
- Software evolves over time. Information you thought was only needed by one library is later needed by another. A good messaging system facilitates convenient data sharing between modules by decoupling data producers from consumers
|
||||||
|
|
||||||
|
- Portability of modules
|
||||||
|
- Again, system architecture and modules evolve over time. If you are using multiple messaging stacks, this evolution becomes more difficult.
|
||||||
|
- ie: Library A was designed to use a messaging protocol with a network stack. What happens if later you now want to run Library A on a bare-metal component?
|
||||||
|
- ie: Library A runs on the same processor as Library B so they share information via an internal method (direct call/api, IPC, etc), what happens if on the next system revision, library A now needs to run on a different processor than library B?
|
||||||
|
|
||||||
|
.. image:: ../assets/img/writeup/unified_messaging_small_uas/module_portability_problem.png
|
||||||
|
:class: dark-light
|
||||||
|
|