significantly improve log search performance

Signed-off-by: huanggze <loganhuang@yunify.com>

pkg/utils/esutil/esutil.go

@@ -0,0 +1,32 @@
+// TODO: refactor
+package esutil
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// KubeSphere uses the layout `yyyy.MM.dd`.
+const layoutISO = "2006.01.02"
+
+func ResolveIndexNames(prefix string, start, end time.Time) string {
+	if end.IsZero() {
+		end = time.Now()
+	}
+
+	// In case of no start time or a broad query range over 30 days, search all indices.
+	if start.IsZero() || end.Sub(start).Hours() > 24*30 {
+		return fmt.Sprintf("%s*", prefix)
+	}
+
+	var indices []string
+	for i := 0; i <= int(end.Sub(start).Hours()/24); i++ {
+		suffix := end.Add(time.Duration(-i) * 24 * time.Hour).Format(layoutISO)
+		indices = append(indices, fmt.Sprintf("%s-%s", prefix, suffix))
+	}
+
+	// If start is after end, ResolveIndexNames returns "".
+	// However, query parameter validation will prevent it from happening at the very beginning (Bad Request).
+	return strings.Join(indices, ",")
+}

pkg/utils/esutil/esutil_test.go

@@ -0,0 +1,56 @@
+package esutil
+
+import (
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"testing"
+	"time"
+)
+
+func TestResolveIndexNames(t *testing.T) {
+	var tests = []struct {
+		prefix   string
+		start    time.Time
+		end      time.Time
+		expected string
+	}{
+		{
+			prefix:   "ks-logstash-log",
+			start:    time.Date(2020, time.February, 1, 0, 0, 0, 0, time.UTC),
+			end:      time.Date(2020, time.February, 1, 0, 0, 0, 0, time.UTC),
+			expected: "ks-logstash-log-2020.02.01",
+		},
+		{
+			prefix:   "ks-logstash-log",
+			start:    time.Date(2020, time.February, 1, 0, 0, 0, 0, time.UTC),
+			end:      time.Date(2020, time.February, 3, 0, 0, 0, 0, time.UTC),
+			expected: "ks-logstash-log-2020.02.03,ks-logstash-log-2020.02.02,ks-logstash-log-2020.02.01",
+		},
+		{
+			prefix:   "ks-logstash-log",
+			end:      time.Date(2020, time.February, 3, 0, 0, 0, 0, time.UTC),
+			expected: "ks-logstash-log*",
+		},
+		{
+			prefix:   "ks-logstash-log",
+			start:    time.Date(2000, time.February, 1, 0, 0, 0, 0, time.UTC),
+			end:      time.Date(2020, time.February, 3, 0, 0, 0, 0, time.UTC),
+			expected: "ks-logstash-log*",
+		},
+		{
+			prefix:   "ks-logstash-log",
+			start:    time.Date(2020, time.February, 3, 0, 0, 0, 0, time.UTC),
+			end:      time.Date(2020, time.February, 1, 0, 0, 0, 0, time.UTC),
+			expected: "",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			result := ResolveIndexNames(test.prefix, test.start, test.end)
+			if diff := cmp.Diff(result, test.expected); diff != "" {
+				t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
+			}
+		})
+	}
+}