[dumper] add column filter #30

conf/mydumper.ini.sample

@@ -17,6 +17,9 @@ chunksize = 128
 # vars= "xx=xx;xx=xx;"
 vars= ""
 
+# Dump some specific tables
+# table = t1,t2
+
 # Use this to use regexp to control what databases to export. These are optional
 [database]
 # regexp = ^(mysql|sys|information_schema|performance_schema)$
@@ -36,3 +39,7 @@ vars= ""
 
 # customer.first_name = CONCAT('Bohu', id)
 # customer.last_name = 'Last'
+
+# Use this to ignore the column to dump.
+[filter]
+# table1.column1 = ignore

src/cmd/config.go

@@ -90,6 +90,27 @@ func parseDumperConfig(file string) (*common.Args, error) {
 		database_invert_regexp = false
 	}
 
+	var filters []string
+	if filters, err = cfg.GetOptions("filter"); err != nil {
+		return nil, err
+	}
+	for _, tblcol := range filters {
+		var table, column string
+		split := strings.Split(tblcol, ".")
+		table = split[0]
+		column = split[1]
+
+		if args.Filters == nil {
+			args.Filters = make(map[string]map[string]string)
+		}
+		if args.Filters[table] == nil {
+			args.Filters[table] = make(map[string]string, 0)
+		}
+		if args.Filters[table][column], err = cfg.GetString("filter", tblcol); err != nil {
+			return nil, err
+		}
+	}
+
 	args.Address = fmt.Sprintf("%s:%d", host, port)
 	args.User = user
 	args.Password = password

src/common/common.go

@@ -41,6 +41,7 @@ type Args struct {
 	OverwriteTables      bool
 	Wheres               map[string]string
 	Selects              map[string]map[string]string
+	Filters              map[string]map[string]string
 
 	// Interval in millisecond.
 	IntervalMs int

src/common/dumper.go

@@ -59,6 +59,11 @@ func dumpTable(log *xlog.Log, conn *Connection, args *Args, database string, tab
 
 		flds := cursor.Fields()
 		for _, fld := range flds {
+			log.Debug("dump -- %#v, %s, %s", args.Filters, table, fld.Name)
+			if _, ok := args.Filters[table][fld.Name]; ok {
+				continue
+			}
+
 			fields = append(fields, fmt.Sprintf("`%s`", fld.Name))
 			replacement, ok := args.Selects[table][fld.Name]
 			if ok {