From ea32e017e579f168d87732893335c38d539ac2f1 Mon Sep 17 00:00:00 2001 From: We-unite <3205135446@qq.com> Date: Wed, 7 Aug 2024 19:08:59 +0800 Subject: Print err in stderr, Find out docker rootfs. When I use godo, error infomation comes along with other output, so change all err report into stderr. And I listen to `pivot_root` sys- call to find out the root file system of dockers. However, I'm afraid of causing too more delay, so don't check rootfs of ppid and record in the pid. Besides, the method to deal with pivot_root is hardcoded, which may causes crush. Shall I listen to the chdir syscall to find out exact cwd? Maybe It's useful to the pivot_root? Next step: Find out appropriate data stracture, and add more file operations to be watched. This task must be completed this week. --- src/deal.go | 51 ++++++++++++++++++++++++++++-------- src/global.go | 6 +++-- src/godo.go | 24 +++++++---------- src/organize.go | 81 ++++++++++++++++++++++++++++++++++----------------------- 4 files changed, 103 insertions(+), 59 deletions(-) (limited to 'src') diff --git a/src/deal.go b/src/deal.go index f2b7d4b..e553174 100644 --- a/src/deal.go +++ b/src/deal.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "syscall" "go.mongodb.org/mongo-driver/bson" @@ -26,7 +27,7 @@ func deal() { var ok bool if err = pidCol.init(dbName, pidColName); err != nil { - fmt.Printf("Error while initing the mongodb: %v\n", err) + fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err) return } err = pidCol.InsertOne(bson.M{ @@ -37,16 +38,16 @@ func deal() { "daemon": true, }) if err != nil { - fmt.Printf("Error while initing the mongodb: %v\n", err) + fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err) return } if err = fdCol.init(dbName, fdColName); err != nil { - fmt.Printf("Error while initing the mongodb: %v\n", err) + fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err) return } if err = fileCol.init(dbName, fileColName); err != nil { - fmt.Printf("Error while initing the mongodb: %v\n", err) + fmt.Fprintf(os.Stderr, "Error while initing the mongodb: %v\n", err) } fmt.Printf("Containerd: %d\n", containerdPid) @@ -73,6 +74,8 @@ func deal() { go fileWrite(cooked) case FILECLOSE: go fileClose(cooked) + case PIVOTROOT: + go pivotRoot(cooked) } } } @@ -85,9 +88,6 @@ func deletePid(cooked Event) { }, }) - // 孩子们需要收容 - // 不必到children里一个个找，直接看ppid即可 - // pidCol.UpdateMany(bson.M{"ppid": cooked.pid}, bson.M{"ppid": 1}) // 在这套逻辑里，孩子是不需要收容的，因为我们根本就不看ppid来工作 // 可以去死了 @@ -98,13 +98,15 @@ func deletePid(cooked Event) { "exit_signal": cooked.exit_signal, }, }) + + // 理论上这里需要关闭所有文件描述符，但为了处理效率，留给后续流程 } func dealNewPid(cooked Event) { // 自身是否已经记录 docRes, err = pidCol.Finddoc(bson.M{"pid": cooked.pid}) if err != nil { - fmt.Printf("Err finding: %v\n", err) + fmt.Fprintf(os.Stderr, "Err finding: %v\n", err) return } @@ -189,7 +191,7 @@ func fileOpen(cooked Event) { // 权限检查过了，不必再查 fdCol.InsertOne(bson.M{ "timestamp": cooked.timestamp, - "fileName": cooked.pathName, + "fileName": cooked.srcPath, "pid": cooked.pid, "fd": cooked.exit_code, "flags": cooked.syscallParam, @@ -212,7 +214,7 @@ func fileClose(cooked Event) { } res["close_timestamp"] = cooked.timestamp if err := fileCol.InsertOne(res); err != nil { - fmt.Printf("Err inserting files: %v\n", err) + fmt.Fprintf(os.Stderr, "Err inserting files: %v\n", err) } } @@ -223,7 +225,7 @@ func fileWrite(cooked Event) { "close_timestamp": bson.M{"$exists": false}, }) if err != nil { - fmt.Printf("Err closing fd %d of pid %d: %v\n", cooked.syscallParam[0], cooked.pid, err) + fmt.Fprintf(os.Stderr, "Err closing fd %d of pid %d: %v\n", cooked.syscallParam[0], cooked.pid, err) } if len(res) == 0 { return @@ -234,3 +236,30 @@ func fileWrite(cooked Event) { "close_timestamp": bson.M{"$exists": false}, }, bson.M{"$push": bson.M{"written": cooked.timestamp}}) } + +func pivotRoot(cooked Event) { + // docker的根目录信息，记录 + docRes, err := pidCol.Finddoc(bson.M{"pid": cooked.pid}) + if err != nil { + fmt.Fprintf(os.Stderr, "Err finding: %v\n", err) + return + } + + if len(docRes) == 0 { + // fork还没到，等一下 + pidCol.InsertOne(bson.M{ + "start_timestamp": cooked.timestamp, + "ppid": cooked.ppid, + "pid": cooked.pid, + "rootfs": "cwd", + }) + } else { + // 读取已有的工作目录 + cwd := docRes[0]["cwd"] + pidCol.UpdateOne(bson.M{"pid": cooked.pid}, bson.M{ + "$set": bson.M{ + "rootfs": cwd, + }, + }) + } +} diff --git a/src/global.go b/src/global.go index b6635c9..349ba6c 100644 --- a/src/global.go +++ b/src/global.go @@ -14,11 +14,12 @@ const ( FILEOPEN FILECLOSE FILEWRITE + PIVOTROOT TYPENUM ) func (et eventType) String() string { - names := []string{"NEWPID", "PIDEXIT", "EXECVE", "FILEOPEN", "FILECLOSE", "FILEWRITE", "TYPENUM"} + names := []string{"NEWPID", "PIDEXIT", "EXECVE", "FILEOPEN", "FILECLOSE", "FILEWRITE", "PIVOTROOT", "TYPENUM"} if et < NEWPID || et > TYPENUM { return "Unknown" } @@ -32,12 +33,13 @@ type Event struct { ppid, parentTgid int syscall int syscallParam [4]uint64 - pathName string argc int argv []string cwd string exit_code uint64 exit_signal int + srcPath string + destPath string } var wg sync.WaitGroup // 掌管协程 diff --git a/src/godo.go b/src/godo.go index 923ef85..a30aa88 100644 --- a/src/godo.go +++ b/src/godo.go @@ -27,14 +27,14 @@ var ( func main() { // 检查用户身份，并添加auditd规则，监听所有syscall if os.Geteuid() != 0 { - fmt.Printf("Err: Please run me as root, %d!\n", os.Getegid()) + fmt.Fprintf(os.Stderr, "Err: Please run me as root, %d!\n", os.Getegid()) return } // 所有的系统调用号与名称的关系 err := figureOutSyscalls() if err != nil { - fmt.Printf("Error figuring out syscall numbers: %v\n", err) + fmt.Fprintf(os.Stderr, "Error figuring out syscall numbers: %v\n", err) } exec.Command("auditctl", "-D").Run() @@ -43,7 +43,7 @@ func main() { var auditCmd *exec.Cmd - pidSyscall := []string{"execve"} + pidSyscall := []string{"execve", "pivot_root"} // // 设置监听规则 for i := 0; i < len(pidSyscall); i++ { auditCmd = exec.Command("auditctl", "-a", "exit,always", "-F", "arch=b64", "-S", pidSyscall[i]) @@ -61,14 +61,10 @@ func main() { // 查找pid containerdPid, err = getPid() if err != nil { - fmt.Printf("Error finding containerd: %v\n", err) + fmt.Fprintf(os.Stderr, "Error finding containerd: %v\n", err) return } - // 创世之神，1号进程 - // 1号进程还是不要在进程树上直接出现了，不然它的小儿子们都会出现 - // /usr/bin/containerd，也就是我们最关注的进程 - // 开始运行，解析命令行参数后监听 if err := fs.Parse(os.Args[1:]); err != nil { log.Fatal(err) @@ -81,8 +77,8 @@ func main() { func coroutine(client *libaudit.AuditClient) { // 各协程至此开始 - rawChan = make(chan interface{}) - cookedChan = make(chan Event) + rawChan = make(chan interface{}, 65536) + cookedChan = make(chan Event, 65536) wg.Add(1) go procWatch() @@ -101,14 +97,14 @@ func coroutine(client *libaudit.AuditClient) { func procWatch() error { ns, err := netlink.NewNetlinkSocket(syscall.NETLINK_CONNECTOR, 12345) if err != nil { - fmt.Printf("Error creating socket: %v\n", err) + fmt.Fprintf(os.Stderr, "Error creating socket: %v\n", err) return err } defer ns.Close() for { res, err := ns.Receive(20) if err != nil { - fmt.Printf("Error recv: %v\n", err) + fmt.Fprintf(os.Stderr, "Error recv: %v\n", err) continue } for i := 0; i < len(res); i++ { @@ -146,7 +142,7 @@ func checkProc(pCooked *Event) { fileName := fmt.Sprintf("/proc/%d/task/%d/cmdline", pCooked.tgid, pCooked.pid) fd, err := os.Open(fileName) if err != nil { - fmt.Printf("Err: %v\n", err) + fmt.Fprintf(os.Stderr, "Err: %v\n", err) return } @@ -162,7 +158,7 @@ func checkProc(pCooked *Event) { fileName = fmt.Sprintf("/proc/%d/task/%d/cwd", pCooked.tgid, pCooked.pid) pCooked.cwd, err = os.Readlink(fileName) if err != nil { - fmt.Printf("Err readlink %s: %v\n", fileName, err) + fmt.Fprintf(os.Stderr, "Err: %v\n", err) pCooked.cwd = "" } } diff --git a/src/organize.go b/src/organize.go index 12119ad..293371b 100644 --- a/src/organize.go +++ b/src/organize.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "regexp" "strconv" "strings" @@ -21,14 +22,15 @@ var ok bool var event Event var pEvent *Event var eventId, argc int -var errs [6]error + +// var errs [6]error // 要用的正则匹配列表 var ( - syscallRegex = regexp.MustCompile(`audit\((\d+\.\d+):(\d+)\).*?syscall=(\d+)(?:.*?exit=([-+]?\d+))?.*?ppid=(\d+) pid=(\d+).*?$`) + syscallRegex = regexp.MustCompile(`audit\((\d+\.\d+):(\d+)\).*?syscall=(\d+)(?:.*?exit=([-+]?\d+))?.*?ppid=(\d+) pid=(\d+).*?subj=(.*?):(.*?):(.*?):(.*?) .*?$`) execveRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): argc=(\d+)`) argsRegex = regexp.MustCompile(`a\d+=("(.*?)"|([0-9a-fA-F]+))`) - pathRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\):.*?name="(.*?)"`) + pathRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): item=(\d+) name="(.*?)"`) cwdRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): cwd="(.*?)"`) proctitleRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\): proctitle=("(.*?)"|([0-9a-fA-F]+))$`) eoeRegex = regexp.MustCompile(`audit\(\d+\.\d+:(\d+)\)`) @@ -47,22 +49,22 @@ func orgnaze() { break } rawEvent = raw.(libaudit.RawAuditMessage) + // fmt.Printf("type=%v msg=%s\n", rawEvent.Type, rawEvent.Data) switch rawEvent.Type { case auparse.AUDIT_SYSCALL: - go syscallRaw(rawEvent) + syscallRaw(rawEvent) case auparse.AUDIT_EXECVE: - go execve(rawEvent) + execve(rawEvent) case auparse.AUDIT_CWD: - go cwd(rawEvent) + cwd(rawEvent) case auparse.AUDIT_PATH: - go path(rawEvent) + path(rawEvent) case auparse.AUDIT_PROCTITLE: - go proctitle(rawEvent) + proctitle(rawEvent) case auparse.AUDIT_EOE: - go eoe(rawEvent) + eoe(rawEvent) default: - // ATTENTION: 这里也需要做防护 } } } @@ -74,28 +76,34 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) { var exit int var a [4]uint64 + var subj [4]string // 捕获基础信息 match := syscallRegex.FindSubmatch(rawEvent.Data) - event.timestamp, errs[0] = getTimeFromStr(string(match[1])) - eventId, errs[1] = strconv.Atoi(string(match[2])) - event.syscall, errs[2] = strconv.Atoi(string(match[3])) + event.timestamp, _ = getTimeFromStr(string(match[1])) + eventId, _ = strconv.Atoi(string(match[2])) + event.syscall, _ = strconv.Atoi(string(match[3])) if string(match[4]) == "" { // exit没捕获到 exit = 0 } else { - exit, errs[3] = strconv.Atoi(string(match[4])) + exit, _ = strconv.Atoi(string(match[4])) + } + event.ppid, _ = strconv.Atoi(string(match[5])) + event.pid, _ = strconv.Atoi(string(match[6])) + + // 几个subj，说不定会有用 + for i := 0; i < 4; i++ { + subj[i] = string(match[7+i]) } - event.ppid, errs[4] = strconv.Atoi(string(match[5])) - event.pid, errs[5] = strconv.Atoi(string(match[6])) // 捕获参数 if !argsRegex.Match(rawEvent.Data) { - fmt.Printf("Error: don't get args in syscall event!\n") + fmt.Fprintf(os.Stderr, "Error: don't get args in syscall event!\n") return } argsMatch := argsRegex.FindAllSubmatch(rawEvent.Data, -1) for i := 0; i < 4; i++ { - a[i], errs[0] = strconv.ParseUint(string(argsMatch[i][3]), 16, 64) + a[i], _ = strconv.ParseUint(string(argsMatch[i][3]), 16, 64) } switch syscallTable[event.syscall] { @@ -128,7 +136,7 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) { argv: make([]string, 0), cwd: "", syscallParam: a, - pathName: "", + srcPath: "", }) case "write": eventTable.Store(eventId, &Event{ @@ -142,7 +150,6 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) { argv: make([]string, 0), cwd: "", syscallParam: a, - // pathName: "", }) case "close": // 文件关闭 @@ -157,8 +164,18 @@ func syscallRaw(rawEvent libaudit.RawAuditMessage) { argv: make([]string, 0), cwd: "", syscallParam: a, - // pathName: "", }) + case "pivot_root": + if subj[2] == "container_runtime_t" { + eventTable.Store(eventId, &Event{ + tag: PIVOTROOT, + timestamp: event.timestamp, + syscall: event.syscall, + ppid: event.ppid, + pid: event.pid, + syscallParam: a, + }) + } } } @@ -168,14 +185,14 @@ func execve(rawEvent libaudit.RawAuditMessage) { } match := execveRegex.FindSubmatch(rawEvent.Data) - eventId, errs[0] = strconv.Atoi(string(match[1])) - argc, errs[1] = strconv.Atoi(string(match[2])) + eventId, _ = strconv.Atoi(string(match[1])) + argc, _ = strconv.Atoi(string(match[2])) tmp, ok = eventTable.Load(eventId) if !ok { return } pEvent = tmp.(*Event) - if errs[0] == nil && errs[1] == nil && argsRegex.Match(rawEvent.Data) { + if argsRegex.Match(rawEvent.Data) { match := argsRegex.FindAllSubmatch(rawEvent.Data, -1) for i := 0; i < argc; i++ { if len(match[i][2]) == 0 { @@ -196,7 +213,7 @@ func cwd(rawEvent libaudit.RawAuditMessage) { } match := cwdRegex.FindSubmatch(rawEvent.Data) - eventId, errs[0] = strconv.Atoi(string(match[1])) + eventId, _ = strconv.Atoi(string(match[1])) tmp, ok = eventTable.Load(eventId) if !ok { return @@ -211,7 +228,7 @@ func proctitle(rawEvent libaudit.RawAuditMessage) { var cmdline string match := proctitleRegex.FindSubmatch(rawEvent.Data) - eventId, errs[0] = strconv.Atoi(string(match[1])) + eventId, _ = strconv.Atoi(string(match[1])) tmp, ok = eventTable.Load(eventId) if !ok { return @@ -236,14 +253,13 @@ func eoe(rawEvent libaudit.RawAuditMessage) { } match := eoeRegex.FindSubmatch(rawEvent.Data) - eventId, errs[0] = strconv.Atoi(string(match[1])) + eventId, _ = strconv.Atoi(string(match[1])) tmp, ok = eventTable.Load(eventId) if !ok { return } cooked := *(tmp.(*Event)) cookedChan <- cooked - // fmt.Printf("Send: %10d\t%v\t%7d\t%7d\n", eventId, cooked.tag, cooked.ppid, cooked.pid) eventTable.Delete(eventId) // 死人别占地 } @@ -252,8 +268,9 @@ func path(rawEvent libaudit.RawAuditMessage) { return } match := pathRegex.FindSubmatch(rawEvent.Data) - eventId, errs[0] = strconv.Atoi(string(match[1])) - name := string(match[2]) + eventId, _ = strconv.Atoi(string(match[1])) + // item, _ := strconv.Atoi(string(match[2])) + name := string(match[3]) tmp, ok = eventTable.Load(eventId) if !ok { @@ -267,8 +284,8 @@ func path(rawEvent libaudit.RawAuditMessage) { } if name[0] == '/' { - pEvent.pathName = name + pEvent.srcPath = name } else { - pEvent.pathName += "/" + name + pEvent.srcPath += "/" + name } } -- cgit v1.2.3-70-g09d2