-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(inthewild): support inTheWild PoCs (#78)
* feat(inthewild): support inTheWild PoCs * docs: update README
- Loading branch information
Showing
8 changed files
with
257 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package commands | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/inconshreveable/log15" | ||
| "github.com/spf13/cobra" | ||
| "github.com/spf13/viper" | ||
| "golang.org/x/xerrors" | ||
|
|
||
| "github.com/vulsio/go-exploitdb/db" | ||
| "github.com/vulsio/go-exploitdb/fetcher" | ||
| "github.com/vulsio/go-exploitdb/models" | ||
| "github.com/vulsio/go-exploitdb/util" | ||
| ) | ||
|
|
||
| var fetchInTheWildCmd = &cobra.Command{ | ||
| Use: "inthewild", | ||
| Short: "Fetch the data of inTheWild Poc", | ||
| Long: `Fetch the data of inTheWild Poc`, | ||
| RunE: fetchInTheWild, | ||
| } | ||
|
|
||
| func init() { | ||
| fetchCmd.AddCommand(fetchInTheWildCmd) | ||
| } | ||
|
|
||
| func fetchInTheWild(_ *cobra.Command, _ []string) (err error) { | ||
| if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { | ||
| return xerrors.Errorf("Failed to SetLogger. err: %w", err) | ||
| } | ||
|
|
||
| driver, locked, err := db.NewDB( | ||
| viper.GetString("dbtype"), | ||
| viper.GetString("dbpath"), | ||
| viper.GetBool("debug-sql"), | ||
| db.Option{}, | ||
| ) | ||
| if err != nil { | ||
| if locked { | ||
| return xerrors.Errorf("Failed to initialize DB. Close DB connection before fetching. err: %w", err) | ||
| } | ||
| return xerrors.Errorf("Failed to open DB. err: %w", err) | ||
| } | ||
|
|
||
| fetchMeta, err := driver.GetFetchMeta() | ||
| if err != nil { | ||
| return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) | ||
| } | ||
| if fetchMeta.OutDated() { | ||
| return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) | ||
| } | ||
| // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. | ||
| if err := driver.UpsertFetchMeta(fetchMeta); err != nil { | ||
| return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) | ||
| } | ||
|
|
||
| log15.Info("Fetching inTheWild Poc Exploit") | ||
| var exploits []models.Exploit | ||
| if exploits, err = fetcher.FetchInTheWild(viper.GetBool("debug-sql")); err != nil { | ||
| return xerrors.Errorf("Failed to fetch InTheWild Exploit. err: %w", err) | ||
| } | ||
| log15.Info("inTheWild Poc Exploit", "count", len(exploits)) | ||
|
|
||
| log15.Info("Insert Exploit into go-exploitdb.", "db", driver.Name()) | ||
| if err := driver.InsertExploit(models.InTheWildType, exploits); err != nil { | ||
| return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) | ||
| } | ||
|
|
||
| fetchMeta.LastFetchedAt = time.Now() | ||
| if err := driver.UpsertFetchMeta(fetchMeta); err != nil { | ||
| return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| package fetcher | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "crypto/md5" | ||
| "database/sql" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "golang.org/x/xerrors" | ||
| "gorm.io/driver/sqlite" | ||
| "gorm.io/gorm" | ||
| "gorm.io/gorm/logger" | ||
|
|
||
| "github.com/vulsio/go-exploitdb/models" | ||
| "github.com/vulsio/go-exploitdb/util" | ||
| ) | ||
|
|
||
| // https://www.sqlite.org/fileformat.html | ||
| var sqlite3HeaderMagic = []byte("SQLite format 3\x00") | ||
|
|
||
| const dbURL string = "https://github.com/gmatuz/inthewilddb/blob/master/inthewild.db?raw=true" | ||
|
|
||
| type exploit struct { | ||
| CVEID string `gorm:"column:exploits.id"` | ||
| Description sql.NullString `gorm:"column:vulns.description"` | ||
| ReferenceURL string `gorm:"column:exploits.referenceURL"` | ||
| TimeStamp time.Time `gorm:"column:exploits.timestamp"` | ||
| Source string `gorm:"column:exploits.source"` | ||
| Type string `gorm:"column:exploits.type"` | ||
| } | ||
|
|
||
| // FetchInTheWild : | ||
| func FetchInTheWild(debugSQL bool) ([]models.Exploit, error) { | ||
| if err := os.MkdirAll(util.CacheDir(), 0700); err != nil { | ||
| return nil, xerrors.Errorf("Failed to mkdir: %w", err) | ||
| } | ||
| dbPath := filepath.Join(util.CacheDir(), "inthewild.db") | ||
| f, err := os.Create(dbPath) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to create inthewild.db in cache directory. err: %w", err) | ||
| } | ||
| defer f.Close() | ||
|
|
||
| res, err := util.FetchURL(dbURL) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to fetch inTheWild DB. err: %w", err) | ||
| } | ||
| if !bytes.Equal(res[:16], sqlite3HeaderMagic) { | ||
| return nil, xerrors.Errorf("Failed to fetch inTheWild DB. err: not sqlite3 database") | ||
| } | ||
| if n, err := f.Write(res); err != nil { | ||
| return nil, xerrors.Errorf("Failed to write db. err: %w", err) | ||
| } else if n != len(res) { | ||
| return nil, xerrors.Errorf("Failed to write db. err: not enough bytes written") | ||
| } | ||
|
|
||
| gormConfig := gorm.Config{ | ||
| DisableForeignKeyConstraintWhenMigrating: true, | ||
| Logger: logger.New( | ||
| log.New(os.Stderr, "\r\n", log.LstdFlags), | ||
| logger.Config{ | ||
| LogLevel: logger.Silent, | ||
| }, | ||
| ), | ||
| } | ||
|
|
||
| if debugSQL { | ||
| gormConfig.Logger = logger.New( | ||
| log.New(os.Stderr, "\r\n", log.LstdFlags), | ||
| logger.Config{ | ||
| SlowThreshold: time.Second, | ||
| LogLevel: logger.Info, | ||
| Colorful: true, | ||
| }, | ||
| ) | ||
| } | ||
| db, err := gorm.Open(sqlite.Open(dbPath), &gormConfig) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to open inTheWild DB. err: %w", err) | ||
| } | ||
| sqlDB, err := db.DB() | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to get DB Object. err : %w", err) | ||
| } | ||
| defer sqlDB.Close() | ||
|
|
||
| rows, err := db. | ||
| Table("exploits"). | ||
| Select("exploits.id, vulns.description, exploits.referenceURL, exploits.timestamp, exploits.source, exploits.type"). | ||
| Joins("LEFT JOIN vulns ON vulns.id = exploits.id"). | ||
| Rows() | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to select exploits. err: %w", err) | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| exploits := []models.Exploit{} | ||
| var exploit exploit | ||
| for rows.Next() { | ||
| if err := rows.Scan(&exploit.CVEID, &exploit.Description, &exploit.ReferenceURL, &exploit.TimeStamp, &exploit.Source, &exploit.Type); err != nil { | ||
| return nil, xerrors.Errorf("Failed to scan rows. err: %w", err) | ||
| } | ||
|
|
||
| // manually correct CVE-ID. ref: https://github.com/gmatuz/inthewilddb/issues/6 | ||
| if exploit.CVEID == "CVE 2020-12271" { | ||
| exploit.CVEID = "CVE-2020-12271" | ||
| } | ||
|
|
||
| exploits = append(exploits, models.Exploit{ | ||
| ExploitType: models.InTheWildType, | ||
| ExploitUniqueID: fmt.Sprintf("%s-%x", models.InTheWildType, md5.Sum([]byte(exploit.CVEID+exploit.ReferenceURL+exploit.Source+exploit.Type))), | ||
| URL: exploit.ReferenceURL, | ||
| Description: exploit.Description.String, | ||
| CveID: exploit.CVEID, | ||
| InTheWild: &models.InTheWild{ | ||
| TimeStamp: exploit.TimeStamp, | ||
| Source: exploit.Source, | ||
| Type: exploit.Type, | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| return exploits, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters