Print err in stderr, Find out docker rootfs.collector

When I use godo, error infomation comes along with other output, so
change all err report into stderr. And I listen to `pivot_root` sys-
call to find out the root file system of dockers. However, I'm afraid
of causing too more delay, so don't check rootfs of ppid and record
in the pid. Besides, the method to deal with pivot_root is hardcoded,
which may causes crush.

Shall I listen to the chdir syscall to find out exact cwd? Maybe It's
useful to the pivot_root?

Next step: Find out appropriate data stracture, and add more file
operations to be watched. This task must be completed this week.

4 files changed, 103 insertions, 59 deletions

diff --git a/src/deal.go b/src/deal.go
index f2b7d4b..e553174 100644
--- a/src/deal.go
+++ b/src/deal.go
@@ -2,6 +2,7 @@ package main
 
 import (
         "fmt"
+        "os"
         "syscall"
 
         "go.mongodb.org/mongo-driver/bson"
@@ -26,7 +27,7 @@ func deal() {
         var ok bool
 
         if err = pidCol.init(dbName, pidColName); err != nil {
-                fmt.Printf("Error while initing the mongodb: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err)
                 return
         }
         err = pidCol.InsertOne(bson.M{
@@ -37,16 +38,16 @@ func deal() {
                 "daemon":   true,
         })
         if err != nil {
-                fmt.Printf("Error while initing the mongodb: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err)
                 return
         }
 
         if err = fdCol.init(dbName, fdColName); err != nil {
-                fmt.Printf("Error while initing the mongodb: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err)
                 return
         }
         if err = fileCol.init(dbName, fileColName); err != nil {
-                fmt.Printf("Error while initing the mongodb: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err)
         }
 
         fmt.Printf("Containerd: %d\n", containerdPid)
@@ -73,6 +74,8 @@ func deal() {
                         go fileWrite(cooked)
                 case FILECLOSE:
                         go fileClose(cooked)
+                case PIVOTROOT:
+                        go pivotRoot(cooked)
                 }
         }
 }
@@ -85,9 +88,6 @@ func deletePid(cooked Event) {
                 },
         })
 
-        // 孩子们需要收容
-        // 不必到children里一个个找，直接看ppid即可
-        // pidCol.UpdateMany(bson.M{"ppid": cooked.pid}, bson.M{"ppid": 1})
         // 在这套逻辑里，孩子是不需要收容的，因为我们根本就不看ppid来工作
 
         // 可以去死了
@@ -98,13 +98,15 @@ func deletePid(cooked Event) {
                         "exit_signal":    cooked.exit_signal,
                 },
         })
+
+        // 理论上这里需要关闭所有文件描述符，但为了处理效率，留给后续流程
 }
 
 func dealNewPid(cooked Event) {
         // 自身是否已经记录
         docRes, err = pidCol.Finddoc(bson.M{"pid": cooked.pid})
         if err != nil {
-                fmt.Printf("Err finding: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Err finding: %v\n", err)
                 return
         }
 
@@ -189,7 +191,7 @@ func fileOpen(cooked Event) {
         // 权限检查过了，不必再查
         fdCol.InsertOne(bson.M{
                 "timestamp": cooked.timestamp,
-                "fileName":  cooked.pathName,
+                "fileName":  cooked.srcPath,
                 "pid":       cooked.pid,
                 "fd":        cooked.exit_code,
                 "flags":     cooked.syscallParam,
@@ -212,7 +214,7 @@ func fileClose(cooked Event) {
         }
         res["close_timestamp"] = cooked.timestamp
         if err := fileCol.InsertOne(res); err != nil {
-                fmt.Printf("Err inserting files: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Err inserting files: %v\n", err)
         }
 }
 
@@ -223,7 +225,7 @@ func fileWrite(cooked Event) {
                 "close_timestamp": bson.M{"$exists": false},
         })
         if err != nil {
-                fmt.Printf("Err closing fd %d of pid %d: %v\n", cooked.syscallParam[0], cooked.pid, err)
+                fmt.Fprintf(os.Stderr, "Err closing fd %d of pid %d: %v\n", cooked.syscallParam[0], cooked.pid, err)
         }
         if len(res) == 0 {
                 return
@@ -234,3 +236,30 @@ func fileWrite(cooked Event) {
                 "close_timestamp": bson.M{"$exists": false},
         }, bson.M{"$push": bson.M{"written": cooked.timestamp}})
 }
+
+func pivotRoot(cooked Event) {
+        // docker的根目录信息，记录
+        docRes, err := pidCol.Finddoc(bson.M{"pid": cooked.pid})
+        if err != nil {
+                fmt.Fprintf(os.Stderr, "Err finding: %v\n", err)
+                return
+        }
+
+        if len(docRes) == 0 {
+                // fork还没到，等一下
+                pidCol.InsertOne(bson.M{
+                        "start_timestamp": cooked.timestamp,
+                        "ppid":            cooked.ppid,
+                        "pid":             cooked.pid,
+                        "rootfs":          "cwd",
+                })
+        } else {
+                // 读取已有的工作目录
+                cwd := docRes[0]["cwd"]
+                pidCol.UpdateOne(bson.M{"pid": cooked.pid}, bson.M{
+                        "$set": bson.M{
+                                "rootfs": cwd,
+                        },
+                })
+        }
+}
diff --git a/src/global.go b/src/global.go
index b6635c9..349ba6c 100644
--- a/src/global.go
+++ b/src/global.go
@@ -14,11 +14,12 @@ const (
         FILEOPEN
         FILECLOSE
         FILEWRITE
+        PIVOTROOT
         TYPENUM
 )
 
 func (et eventType) String() string {
-        names := []string{"NEWPID", "PIDEXIT", "EXECVE", "FILEOPEN", "FILECLOSE", "FILEWRITE", "TYPENUM"}
+        names := []string{"NEWPID", "PIDEXIT", "EXECVE", "FILEOPEN", "FILECLOSE", "FILEWRITE", "PIVOTROOT", "TYPENUM"}
         if et < NEWPID || et > TYPENUM {
                 return "Unknown"
         }
@@ -32,12 +33,13 @@ type Event struct {
         ppid, parentTgid int
         syscall          int
         syscallParam     [4]uint64
-        pathName         string
         argc             int
         argv             []string
         cwd              string
         exit_code        uint64
         exit_signal      int
+        srcPath          string
+        destPath         string
 }
 
 var wg sync.WaitGroup        // 掌管协程
diff --git a/src/godo.go b/src/godo.go
index 923ef85..a30aa88 100644
--- a/src/godo.go
+++ b/src/godo.go
@@ -27,14 +27,14 @@ var (
 func main() {
         // 检查用户身份，并添加auditd规则，监听所有syscall
         if os.Geteuid() != 0 {
-                fmt.Printf("Err: Please run me as root, %d!\n", os.Getegid())
+                fmt.Fprintf(os.Stderr, "Err: Please run me as root, %d!\n", os.Getegid())
                 return
         }
 
         // 所有的系统调用号与名称的关系
         err := figureOutSyscalls()
         if err != nil {
-                fmt.Printf("Error figuring out syscall numbers: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error figuring out syscall numbers: %v\n", err)
         }
 
         exec.Command("auditctl", "-D").Run()
@@ -43,7 +43,7 @@ func main() {
 
         var auditCmd *exec.Cmd
 
-        pidSyscall := []string{"execve"}
+        pidSyscall := []string{"execve", "pivot_root"}
         // // 设置监听规则
         for i := 0; i < len(pidSyscall); i++ {
                 auditCmd = exec.Command("auditctl", "-a", "exit,always", "-F", "arch=b64", "-S", pidSyscall[i])
@@ -61,14 +61,10 @@ func main() {
         // 查找pid
         containerdPid, err = getPid()
         if err != nil {
-                fmt.Printf("Error finding containerd: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error finding containerd: %v\n", err)
                 return
         }
 
-        // 创世之神，1号进程
-        // 1号进程还是不要在进程树上直接出现了，不然它的小儿子们都会出现
-        // /usr/bin/containerd，也就是我们最关注的进程
-
         // 开始运行，解析命令行参数后监听
         if err := fs.Parse(os.Args[1:]); err != nil {
                 log.Fatal(err)
@@ -81,8 +77,8 @@ func main() {
 
 func coroutine(client *libaudit.AuditClient) {
         // 各协程至此开始
-        rawChan = make(chan interface{})
-        cookedChan = make(chan Event)
+        rawChan = make(chan interface{}, 65536)
+        cookedChan = make(chan Event, 65536)
 
         wg.Add(1)
         go procWatch()
@@ -101,14 +97,14 @@ func coroutine(client *libaudit.AuditClient) {
 func procWatch() error {
         ns, err := netlink.NewNetlinkSocket(syscall.NETLINK_CONNECTOR, 12345)
         if err != nil {
-                fmt.Printf("Error creating socket: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Error creating socket: %v\n", err)
                 return err
         }
         defer ns.Close()
         for {
                 res, err := ns.Receive(20)
                 if err != nil {
-                        fmt.Printf("Error recv: %v\n", err)
+                        fmt.Fprintf(os.Stderr, "Error recv: %v\n", err)
                         continue
                 }
                 for i := 0; i < len(res); i++ {
@@ -146,7 +142,7 @@ func checkProc(pCooked *Event) {
         fileName := fmt.Sprintf("/proc/%d/task/%d/cmdline", pCooked.tgid, pCooked.pid)
         fd, err := os.Open(fileName)
         if err != nil {
-                fmt.Printf("Err: %v\n", err)
+                fmt.Fprintf(os.Stderr, "Err: %v\n", err)
                 return
         }
 
@@ -162,7 +158,7 @@ func checkProc(pCooked *Event) {
         fileName = fmt.Sprintf("/proc/%d/task/%d/cwd", pCooked.tgid, pCooked.pid)
         pCooked.cwd, err = os.Readlink(fileName)
         if err != nil {
-                fmt.Printf("Err readlink %s: %v\n", fileName, err)
+                fmt.Fprintf(os.Stderr, "Err: %v\n", err)
                 pCooked.cwd = ""
         }
 }
diff --git a/src/organize.go b/src/organize.go
index 12119ad..293371b 100644
--- a/src/organize.go
+++ b/src/organize.go
@@ -2,6 +2,7 @@ package main
 
 import (
         "fmt"
+        "os"
         "regexp"
         "strconv"
         "strings"
@@ -21,14 +22,15 @@ var ok bool
 var event Event
 var pEvent *Event
 var eventId, argc int
-var errs [6]error
+
+// var errs [6]error
 
 // 要用的正则匹配列表
 var (
-        syscallRegex   = regexp.MustCompile(`audit\((\d+\.\d+):(\d+)\).*?syscall=(\d+)(?:.*?exit=([-+]?\d+))?.*?ppid=(\d+) pid=(\d+).*?$`)
+        syscallRegex   = regexp.MustCompile(`audit\((\d+\.\d+):(\d+)\).*?syscall=(\d+)(?:.*?exit=([-+]?\d+))?.*?ppid=(\d+) pid=(\d+).*?subj=(.*?):(.*?):(.*?):(.*?) .*?$`)
         execveRegex    = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): argc=(\d+)`)
         argsRegex      = regexp.MustCompile(`a\d+=("(.*?)"|([0-9a-fA-F]+))`)
-        pathRegex      = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\):.*?name="(.*?)"`)
+        pathRegex      = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): item=(\d+) name="(.*?)"`)
         cwdRegex       = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\):  cwd="(.*?)"`)
         proctitleRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): proctitle=("(.*?)"|([0-9a-fA-F]+))$`)
         eoeRegex       = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\)`)
@@ -47,22 +49,22 @@ func orgnaze() {
                         break
                 }
                 rawEvent = raw.(libaudit.RawAuditMessage)
+                // fmt.Printf("type=%v msg=%s\n", rawEvent.Type, rawEvent.Data)
 
                 switch rawEvent.Type {
                 case auparse.AUDIT_SYSCALL:
-                        go syscallRaw(rawEvent)
+                        syscallRaw(rawEvent)
                 case auparse.AUDIT_EXECVE:
-                        go execve(rawEvent)
+                        execve(rawEvent)
                 case auparse.AUDIT_CWD:
-                        go cwd(rawEvent)
+                        cwd(rawEvent)
                 case auparse.AUDIT_PATH:
-                        go path(rawEvent)
+                        path(rawEvent)
                 case auparse.AUDIT_PROCTITLE:
-                        go proctitle(rawEvent)
+                        proctitle(rawEvent)
                 case auparse.AUDIT_EOE:
-                        go eoe(rawEvent)
+                        eoe(rawEvent)
                 default:
-                        // ATTENTION: 这里也需要做防护
                 }
         }
 }
@@ -74,28 +76,34 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) {
 
         var exit int
         var a [4]uint64
+        var subj [4]string
         // 捕获基础信息
         match := syscallRegex.FindSubmatch(rawEvent.Data)
-        event.timestamp, errs[0] = getTimeFromStr(string(match[1]))
-        eventId, errs[1] = strconv.Atoi(string(match[2]))
-        event.syscall, errs[2] = strconv.Atoi(string(match[3]))
+        event.timestamp, _ = getTimeFromStr(string(match[1]))
+        eventId, _ = strconv.Atoi(string(match[2]))
+        event.syscall, _ = strconv.Atoi(string(match[3]))
         if string(match[4]) == "" {
                 // exit没捕获到
                 exit = 0
         } else {
-                exit, errs[3] = strconv.Atoi(string(match[4]))
+                exit, _ = strconv.Atoi(string(match[4]))
+        }
+        event.ppid, _ = strconv.Atoi(string(match[5]))
+        event.pid, _ = strconv.Atoi(string(match[6]))
+
+        // 几个subj，说不定会有用
+        for i := 0; i < 4; i++ {
+                subj[i] = string(match[7+i])
         }
-        event.ppid, errs[4] = strconv.Atoi(string(match[5]))
-        event.pid, errs[5] = strconv.Atoi(string(match[6]))
 
         // 捕获参数
         if !argsRegex.Match(rawEvent.Data) {
-                fmt.Printf("Error: don't get args in syscall event!\n")
+                fmt.Fprintf(os.Stderr, "Error: don't get args in syscall event!\n")
                 return
         }
         argsMatch := argsRegex.FindAllSubmatch(rawEvent.Data, -1)
         for i := 0; i < 4; i++ {
-                a[i], errs[0] = strconv.ParseUint(string(argsMatch[i][3]), 16, 64)
+                a[i], _ = strconv.ParseUint(string(argsMatch[i][3]), 16, 64)
         }
 
         switch syscallTable[event.syscall] {
@@ -128,7 +136,7 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) {
                         argv:         make([]string, 0),
                         cwd:          "",
                         syscallParam: a,
-                        pathName:     "",
+                        srcPath:      "",
                 })
         case "write":
                 eventTable.Store(eventId, &Event{
@@ -142,7 +150,6 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) {
                         argv:         make([]string, 0),
                         cwd:          "",
                         syscallParam: a,
-                        // pathName:     "",
                 })
         case "close":
                 // 文件关闭
@@ -157,8 +164,18 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) {
                         argv:         make([]string, 0),
                         cwd:          "",
                         syscallParam: a,
-                        // pathName:     "",
                 })
+        case "pivot_root":
+                if subj[2] == "container_runtime_t" {
+                        eventTable.Store(eventId, &Event{
+                                tag:          PIVOTROOT,
+                                timestamp:    event.timestamp,
+                                syscall:      event.syscall,
+                                ppid:         event.ppid,
+                                pid:          event.pid,
+                                syscallParam: a,
+                        })
+                }
         }
 }
 
@@ -168,14 +185,14 @@ func execve(rawEvent libaudit.RawAuditMessage) {
         }
 
         match := execveRegex.FindSubmatch(rawEvent.Data)
-        eventId, errs[0] = strconv.Atoi(string(match[1]))
-        argc, errs[1] = strconv.Atoi(string(match[2]))
+        eventId, _ = strconv.Atoi(string(match[1]))
+        argc, _ = strconv.Atoi(string(match[2]))
         tmp, ok = eventTable.Load(eventId)
         if !ok {
                 return
         }
         pEvent = tmp.(*Event)
-        if errs[0] == nil && errs[1] == nil && argsRegex.Match(rawEvent.Data) {
+        if argsRegex.Match(rawEvent.Data) {
                 match := argsRegex.FindAllSubmatch(rawEvent.Data, -1)
                 for i := 0; i < argc; i++ {
                         if len(match[i][2]) == 0 {
@@ -196,7 +213,7 @@ func cwd(rawEvent libaudit.RawAuditMessage) {
         }
 
         match := cwdRegex.FindSubmatch(rawEvent.Data)
-        eventId, errs[0] = strconv.Atoi(string(match[1]))
+        eventId, _ = strconv.Atoi(string(match[1]))
         tmp, ok = eventTable.Load(eventId)
         if !ok {
                 return
@@ -211,7 +228,7 @@ func proctitle(rawEvent libaudit.RawAuditMessage) {
 
         var cmdline string
         match := proctitleRegex.FindSubmatch(rawEvent.Data)
-        eventId, errs[0] = strconv.Atoi(string(match[1]))
+        eventId, _ = strconv.Atoi(string(match[1]))
         tmp, ok = eventTable.Load(eventId)
         if !ok {
                 return
@@ -236,14 +253,13 @@ func eoe(rawEvent libaudit.RawAuditMessage) {
         }
 
         match := eoeRegex.FindSubmatch(rawEvent.Data)
-        eventId, errs[0] = strconv.Atoi(string(match[1]))
+        eventId, _ = strconv.Atoi(string(match[1]))
         tmp, ok = eventTable.Load(eventId)
         if !ok {
                 return
         }
         cooked := *(tmp.(*Event))
         cookedChan <- cooked
-        // fmt.Printf("Send: %10d\t%v\t%7d\t%7d\n", eventId, cooked.tag, cooked.ppid, cooked.pid)
         eventTable.Delete(eventId) // 死人别占地
 }
 
@@ -252,8 +268,9 @@ func path(rawEvent libaudit.RawAuditMessage) {
                 return
         }
         match := pathRegex.FindSubmatch(rawEvent.Data)
-        eventId, errs[0] = strconv.Atoi(string(match[1]))
-        name := string(match[2])
+        eventId, _ = strconv.Atoi(string(match[1]))
+        // item, _ := strconv.Atoi(string(match[2]))
+        name := string(match[3])
 
         tmp, ok = eventTable.Load(eventId)
         if !ok {
@@ -267,8 +284,8 @@ func path(rawEvent libaudit.RawAuditMessage) {
         }
 
         if name[0] == '/' {
-                pEvent.pathName = name
+                pEvent.srcPath = name
         } else {
-                pEvent.pathName += "/" + name
+                pEvent.srcPath += "/" + name
         }
 }