From 173fd73974cb34e5d040e9c3acde00ea10b0e050 Mon Sep 17 00:00:00 2001 From: Matthew McNew Date: Thu, 7 Nov 2024 16:28:03 -0600 Subject: [PATCH] Recommend default_toast_compression=lz4 & jit=off - Added to "misc" group - Only provide recommendation on supported versions --- pkg/pgtune/misc.go | 79 ++++++++++------ pkg/pgtune/misc_test.go | 64 ++++++++++++- pkg/tstune/tuner.go | 23 +++-- pkg/tstune/tuner_test.go | 192 ++++++++++++++++++++++++--------------- 4 files changed, 249 insertions(+), 109 deletions(-) diff --git a/pkg/pgtune/misc.go b/pkg/pgtune/misc.go index 9ca4314..0d7137b 100644 --- a/pkg/pgtune/misc.go +++ b/pkg/pgtune/misc.go @@ -20,6 +20,10 @@ const ( AutovacuumNaptimeKey = "autovacuum_naptime" EffectiveIOKey = "effective_io_concurrency" // linux only + // nonnumeric + DefaultToastCompression = "default_toast_compression" + Jit = "jit" + checkpointDefault = "0.9" statsTargetDefault = "100" randomPageCostDefault = "1.1" @@ -31,6 +35,8 @@ const ( // https://www.postgresql.org/message-id/20210422195232.GA25061%40momjian.us effectiveIODefaultOldVersions = "200" effectiveIODefault = "256" + lz4Compression = "lz4" + off = "off" // If you want to lower this value, consider that Patroni will not accept anything less than 25 as // a valid max_connections and will replace it with 100, per @@ -59,12 +65,18 @@ func getMaxConns(totalMemory uint64) uint64 { } } +func getValueForVersion(currentVersion string, oldVersions []string, oldVersionValue, newVersionValue string) string { + for _, ov := range oldVersions { + if ov == currentVersion { + return oldVersionValue + } + } + return newVersionValue +} + func getEffectiveIOConcurrency(pgMajorVersion string) string { switch pgMajorVersion { - case pgutils.MajorVersion96, - pgutils.MajorVersion10, - pgutils.MajorVersion11, - pgutils.MajorVersion12: + case pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12: return effectiveIODefaultOldVersions } return effectiveIODefault @@ -90,7 +102,10 @@ var MiscKeys = []string{ MaxLocksPerTxKey, AutovacuumMaxWorkersKey, AutovacuumNaptimeKey, - EffectiveIOKey, + DefaultToastCompression, + Jit, + + EffectiveIOKey, //linux only } // MiscRecommender gives recommendations for MiscKeys based on system resources. @@ -113,20 +128,38 @@ func (r *MiscRecommender) IsAvailable() bool { // Recommend returns the recommended PostgreSQL formatted value for the conf // file for a given key. func (r *MiscRecommender) Recommend(key string) string { - var val string - if key == CheckpointKey { - val = checkpointDefault - } else if key == StatsTargetKey { - val = statsTargetDefault - } else if key == MaxConnectionsKey { - conns := getMaxConns(r.totalMemory) + switch key { + case CheckpointKey: + return checkpointDefault + case StatsTargetKey: + return statsTargetDefault + case AutovacuumMaxWorkersKey: + return autovacuumMaxWorkersDefault + case AutovacuumNaptimeKey: + return autovacuumNaptimeDefault + case RandomPageCostKey: + return randomPageCostDefault + case EffectiveIOKey: + return getValueForVersion(r.pgMajorVersion, []string{ + pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12}, + effectiveIODefaultOldVersions, effectiveIODefault, + ) + case DefaultToastCompression: + return getValueForVersion(r.pgMajorVersion, []string{ + pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12, pgutils.MajorVersion13}, + NoRecommendation, lz4Compression, + ) + case Jit: + return getValueForVersion(r.pgMajorVersion, []string{ + pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11}, + NoRecommendation, off, + ) + case MaxConnectionsKey: if r.maxConns != 0 { - conns = r.maxConns + return fmt.Sprintf("%d", r.maxConns) } - val = fmt.Sprintf("%d", conns) - } else if key == RandomPageCostKey { - val = randomPageCostDefault - } else if key == MaxLocksPerTxKey { + return fmt.Sprintf("%d", getMaxConns(r.totalMemory)) + case MaxLocksPerTxKey: for i := len(maxLocksValues) - 1; i >= 1; i-- { limit := uint64(math.Pow(2.0, float64(2+i))) if r.totalMemory >= limit*parse.Gigabyte { @@ -134,16 +167,10 @@ func (r *MiscRecommender) Recommend(key string) string { } } return maxLocksValues[0] - } else if key == AutovacuumMaxWorkersKey { - val = autovacuumMaxWorkersDefault - } else if key == AutovacuumNaptimeKey { - val = autovacuumNaptimeDefault - } else if key == EffectiveIOKey { - val = getEffectiveIOConcurrency(r.pgMajorVersion) - } else { - val = NoRecommendation } - return val + + //unknown key no recommendation + return NoRecommendation } // MiscSettingsGroup is the SettingsGroup to represent settings that do not fit in other SettingsGroups. diff --git a/pkg/pgtune/misc_test.go b/pkg/pgtune/misc_test.go index 65a9929..e71e713 100644 --- a/pkg/pgtune/misc_test.go +++ b/pkg/pgtune/misc_test.go @@ -147,6 +147,68 @@ func TestGetEffectiveIOConcurrency(t *testing.T) { } } +func TestDefaultToastCompression(t *testing.T) { + cases := []struct { + pgMajorVersions []string + want string + }{ + { + []string{pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12, pgutils.MajorVersion13}, + NoRecommendation, + }, + { + []string{pgutils.MajorVersion14, pgutils.MajorVersion15, pgutils.MajorVersion16, pgutils.MajorVersion17, + "18", //future versions + }, + "lz4", + }, + } + for _, c := range cases { + for _, v := range c.pgMajorVersions { + t.Run("default_toast_compression:"+v, func(t *testing.T) { + r := NewMiscRecommender(1000, 32, v) + + rec := r.Recommend(DefaultToastCompression) + if rec != c.want { + t.Errorf("wanted %s got: %s", c.want, rec) + } + + }) + } + } +} + +func TestJIT(t *testing.T) { + cases := []struct { + pgMajorVersions []string + want string + }{ + { + []string{pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11}, + NoRecommendation, + }, + { + []string{pgutils.MajorVersion12, pgutils.MajorVersion13, pgutils.MajorVersion14, pgutils.MajorVersion15, pgutils.MajorVersion16, pgutils.MajorVersion17, + "18", //future versions + }, + "off", + }, + } + for _, c := range cases { + for _, v := range c.pgMajorVersions { + t.Run("default_toast_compression:"+v, func(t *testing.T) { + r := NewMiscRecommender(1000, 32, v) + + rec := r.Recommend(Jit) + if rec != c.want { + t.Errorf("wanted %s got: %s", c.want, rec) + } + + }) + } + } +} + func TestNewMiscRecommender(t *testing.T) { for i := 0; i < 1000000; i++ { mem := rand.Uint64() @@ -173,7 +235,7 @@ func TestNewMiscRecommender(t *testing.T) { func TestMiscRecommenderRecommend(t *testing.T) { for totalMemory, outerMatrix := range miscSettingsMatrix { for maxConns, matrix := range outerMatrix { - r := &MiscRecommender{totalMemory, maxConns, pgutils.MajorVersion12} + r := &MiscRecommender{totalMemory, maxConns, pgutils.MajorVersion10} testRecommender(t, r, MiscKeys, matrix) } } diff --git a/pkg/tstune/tuner.go b/pkg/tstune/tuner.go index 84dfa69..ebe8f78 100644 --- a/pkg/tstune/tuner.go +++ b/pkg/tstune/tuner.go @@ -444,6 +444,21 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar rv := pgtune.GetFloatParser(recommender) + // get and parse our recommendation; fail if for we can't + rec := recommender.Recommend(k) + + switch { + case rec == pgtune.NoRecommendation: + // don't bother adding it to the map. no recommendation + continue + case r.commented: + show[k] = true + case r.value == rec: + // don't bother adding it to the map. no recommendation + continue + + } + // parse the value already there; if unparseable, should show our rec curr, err := rv.ParseFloat(k, r.value) if err != nil { @@ -451,19 +466,13 @@ func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunablePar continue } - // get and parse our recommendation; fail if for we can't - rec := recommender.Recommend(k) - if rec == pgtune.NoRecommendation { - // don't bother adding it to the map. no recommendation - continue - } target, err := rv.ParseFloat(k, rec) if err != nil { return nil, fmt.Errorf("unexpected parsing problem: %v", err) } // only show if our recommendation is significantly different, or config is commented - if !isCloseEnough(curr, target, fudgeFactor) || r.commented { + if !isCloseEnough(curr, target, fudgeFactor) { show[k] = true } } diff --git a/pkg/tstune/tuner_test.go b/pkg/tstune/tuner_test.go index 46c0640..d1e83d6 100644 --- a/pkg/tstune/tuner_test.go +++ b/pkg/tstune/tuner_test.go @@ -938,57 +938,96 @@ func TestCheckIfShouldShowSetting(t *testing.T) { } for _, c := range cases { - reset() - // change those keys who are supposed to be commented out - for _, k := range c.commented { - c.parseResults[k].commented = true - } - // change values, but still within fudge factor so it shouldn't be shown - for _, k := range c.okFudge { - temp, err := parse.PGFormatToBytes(c.parseResults[k].value) - if err != nil { - t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + t.Run(c.desc, func(t *testing.T) { + + reset() + // change those keys who are supposed to be commented out + for _, k := range c.commented { + c.parseResults[k].commented = true } - temp = temp + uint64(float64(temp)*(fudgeFactor-.01)) - c.parseResults[k].value = parse.BytesToPGFormat(temp) - } - // change values to higher fudge factor, so it should be shown - for _, k := range c.highFudge { - temp, err := parse.PGFormatToBytes(c.parseResults[k].value) - if err != nil { - t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + // change values, but still within fudge factor so it shouldn't be shown + for _, k := range c.okFudge { + temp, err := parse.PGFormatToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp + uint64(float64(temp)*(fudgeFactor-.01)) + c.parseResults[k].value = parse.BytesToPGFormat(temp) } - temp = temp + uint64(float64(temp)*(fudgeFactor+.01)) - c.parseResults[k].value = parse.BytesToPGFormat(temp) - } - // change values to lower fudge factor, so it should be shown - for _, k := range c.lowFudge { - temp, err := parse.PGFormatToBytes(c.parseResults[k].value) - if err != nil { - t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + // change values to higher fudge factor, so it should be shown + for _, k := range c.highFudge { + temp, err := parse.PGFormatToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp + uint64(float64(temp)*(fudgeFactor+.01)) + c.parseResults[k].value = parse.BytesToPGFormat(temp) } - temp = temp - uint64(float64(temp)*(fudgeFactor+.01)) - c.parseResults[k].value = parse.BytesToPGFormat(temp) - } - mr := pgtune.NewMemoryRecommender(8*parse.Gigabyte, 1, 20) - show, err := checkIfShouldShowSetting(pgtune.MemoryKeys, c.parseResults, mr) - if len(c.errMsg) > 0 { - - } else if err != nil { - t.Errorf("%s: unexpected err: %v", c.desc, err) - } else { - if got := len(show); got != len(c.want) { - t.Errorf("%s: incorrect show length: got %d want %d", c.desc, got, len(c.want)) + // change values to lower fudge factor, so it should be shown + for _, k := range c.lowFudge { + temp, err := parse.PGFormatToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp - uint64(float64(temp)*(fudgeFactor+.01)) + c.parseResults[k].value = parse.BytesToPGFormat(temp) } - for _, k := range c.want { - if _, ok := show[k]; !ok { - t.Errorf("%s: key not found: %s", c.desc, k) + mr := pgtune.NewMemoryRecommender(8*parse.Gigabyte, 1, 20) + + show, err := checkIfShouldShowSetting(pgtune.MemoryKeys, c.parseResults, mr) + if len(c.errMsg) > 0 { + + } else if err != nil { + t.Errorf("%s: unexpected err: %v", c.desc, err) + } else { + if got := len(show); got != len(c.want) { + t.Errorf("%s: incorrect show length: got %d want %d", c.desc, got, len(c.want)) + } + for _, k := range c.want { + if _, ok := show[k]; !ok { + t.Errorf("%s: key not found: %s", c.desc, k) + } } } - } + }) } } +func TestCheckIfShouldShowSettingNonNumeric(t *testing.T) { + mr := pgtune.NewMiscRecommender(1000, 32, "18") + + show, err := checkIfShouldShowSetting([]string{pgtune.DefaultToastCompression, pgtune.Jit}, + map[string]*tunableParseResult{ + pgtune.DefaultToastCompression: { + idx: 0, + commented: false, + key: pgtune.SharedBuffersKey, + value: "pgz", + }, + pgtune.Jit: { + idx: 1, + commented: false, + key: pgtune.SharedBuffersKey, + value: "off", + }, + }, + mr, + ) + + if err != nil { + t.Errorf("%s: unexpected err: %v", t.Name(), err) + } + + if len(show) != 1 { + t.Errorf("%s: incorrect show length: got %d want %d", t.Name(), len(show), 1) + } + + if show[pgtune.DefaultToastCompression] != true { + t.Errorf("exepcted %s to be shown", pgtune.DefaultToastCompression) + } + +} + func TestCheckIfShouldShowSettingErr(t *testing.T) { keys := []string{"foo"} parseResults := map[string]*tunableParseResult{ @@ -1166,7 +1205,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { input: " \ny\n", wantStatements: 3, // intro remark + current label + recommend label wantPrompts: 2, // first input is blank - wantPrints: 9, + wantPrints: 5, successMsg: "memory settings will be updated", shouldErr: false, }, @@ -1275,7 +1314,7 @@ func TestTunerProcessSettingsGroup(t *testing.T) { desc: "bgwriter correct", ts: pgtune.GetSettingsGroup(pgtune.BgwriterLabel, config), profile: pgtune.PromscaleProfile, - lines: []string{"bgwriter_flush_after = 0"}, + lines: []string{"bgwriter_flush_after = 9"}, // will change to 0 input: "y\n", wantStatements: 3, // intro remark + current label + recommend label wantPrompts: 1, @@ -1287,46 +1326,49 @@ func TestTunerProcessSettingsGroup(t *testing.T) { } for _, c := range cases { - tuner := newTunerWithDefaultFlagsForInputs(t, c.input, c.lines) + t.Run(c.desc, func(t *testing.T) { - err := tuner.processSettingsGroup(c.ts, c.profile) - if err != nil && !c.shouldErr { - t.Errorf("%s: unexpected error: %v", c.desc, err) - } else if err == nil && c.shouldErr { - t.Errorf("%s: unexpected lack of error", c.desc) - } + tuner := newTunerWithDefaultFlagsForInputs(t, c.input, c.lines) - tp := tuner.handler.p.(*testPrinter) - if got := strings.ToUpper(strings.TrimSpace(tp.statements[0])[:1]); got != strings.ToUpper(c.ts.Label()[:1]) { - t.Errorf("%s: label not capitalized in first statement: got %s want %s", c.desc, got, strings.ToUpper(c.ts.Label()[:1])) - } + err := tuner.processSettingsGroup(c.ts, c.profile) + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of error", c.desc) + } - if got := tp.statementCalls; got != c.wantStatements { - t.Errorf("%s: incorrect number of statements: got %d want %d", c.desc, got, c.wantStatements) - } + tp := tuner.handler.p.(*testPrinter) + if got := strings.ToUpper(strings.TrimSpace(tp.statements[0])[:1]); got != strings.ToUpper(c.ts.Label()[:1]) { + t.Errorf("%s: label not capitalized in first statement: got %s want %s", c.desc, got, strings.ToUpper(c.ts.Label()[:1])) + } - if got := tp.promptCalls; got != c.wantPrompts { - t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.wantPrompts) - } + if got := tp.statementCalls; got != c.wantStatements { + t.Errorf("%s: incorrect number of statements: got %d want %d", c.desc, got, c.wantStatements) + } - out := tuner.handler.out.(*testWriter) + if got := tp.promptCalls; got != c.wantPrompts { + t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.wantPrompts) + } - if got := len(out.lines); got != c.wantPrints { - t.Errorf("%s: incorrect number of prints: got %d want %d", c.desc, got, c.wantPrints) - } + out := tuner.handler.out.(*testWriter) - if got := tp.errorCalls; got != c.wantErrors { - t.Errorf("%s: incorrect number of errors: got %d want %d", c.desc, got, c.wantErrors) - } else if len(c.successMsg) > 0 { - if got := tp.successCalls; got != 1 { - t.Errorf("%s: incorrect number of successes: got %d want %d", c.desc, got, 1) + if got := len(out.lines); got != c.wantPrints { + t.Errorf("%s: incorrect number of prints: got %d want %d", c.desc, got, c.wantPrints) } - if got := tp.successes[0]; got != c.successMsg { - t.Errorf("%s: incorrect success message: got\n%s\nwant\n%s\n", c.desc, got, c.successMsg) + + if got := tp.errorCalls; got != c.wantErrors { + t.Errorf("%s: incorrect number of errors: got %d want %d", c.desc, got, c.wantErrors) + } else if len(c.successMsg) > 0 { + if got := tp.successCalls; got != 1 { + t.Errorf("%s: incorrect number of successes: got %d want %d", c.desc, got, 1) + } + if got := tp.successes[0]; got != c.successMsg { + t.Errorf("%s: incorrect success message: got\n%s\nwant\n%s\n", c.desc, got, c.successMsg) + } + } else if tp.successCalls > 0 { + t.Errorf("%s: got success without expecting it: %s", c.desc, tp.successes[0]) } - } else if tp.successCalls > 0 { - t.Errorf("%s: got success without expecting it: %s", c.desc, tp.successes[0]) - } + }) } }