diff --git a/conf/cache.go b/conf/cache.go new file mode 100644 index 0000000..c5f5487 --- /dev/null +++ b/conf/cache.go @@ -0,0 +1,7 @@ +package conf + +type CacheSettingsYaml struct { + MaxAge uint `yaml:"maxAge"` + NotModifiedResponseUsingLastModified bool `yaml:"notModifiedUsingLastModified"` + NotModifiedResponseUsingETags bool `yaml:"notModifiedUsingETags"` +} diff --git a/conf/zone.go b/conf/zone.go index ace87ea..c417d87 100644 --- a/conf/zone.go +++ b/conf/zone.go @@ -3,16 +3,18 @@ package conf import "golang.captainalm.com/GOPackageHeaderServer/outputMeta" type ZoneYaml struct { - Name string `yaml:"name"` - Domains []string `yaml:"domains"` - CssURL string `yaml:"cssURL"` - HavePageContents bool `yaml:"havePageContents"` - BasePath string `yaml:"basePath"` - UsernameProvided bool `yaml:"usernameProvided"` - Username string `yaml:"username"` - BasePrefixURL string `yaml:"basePrefixURL"` - SuffixDirectoryURL string `yaml:"suffixDirectoryURL"` - SuffixFileURL string `yaml:"suffixFileURL"` + Name string `yaml:"name"` + Domains []string `yaml:"domains"` + CssURL string `yaml:"cssURL"` + HavePageContents bool `yaml:"havePageContents"` + BasePath string `yaml:"basePath"` + UsernameProvided bool `yaml:"usernameProvided"` + Username string `yaml:"username"` + BasePrefixURL string `yaml:"basePrefixURL"` + SuffixDirectoryURL string `yaml:"suffixDirectoryURL"` + SuffixFileURL string `yaml:"suffixFileURL"` + RangeSupported bool `yaml:"rangeSupported"` + CacheSettings CacheSettingsYaml `yaml:"cacheSettings"` } func (zy ZoneYaml) GetPackageMetaTagOutputter() *outputMeta.PackageMetaTagOutputter { diff --git a/config.example.yml b/config.example.yml index 2488805..8c9235e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -16,3 +16,9 @@ zones: #An array of zones username: "captain-alm" #The username to append to the start of a path under the prefix suffixDirectoryURL: "src/branch/master{/dir}" #The suffix location of the main branch for directory usage suffixFileURL: "src/branch/master{/dir}/{file}#L{line}" #The suffix location of the main branch for file usage + rangeSupported: true #Are range requests supported + cacheSettings: #Cache settings + maxAge: 0 #The maximum age of the cache + notModifiedUsingLastModified: true #Are the conditional headers attached to Last-Modified used to work out if to send a 304 Cache Redirect + notModifiedUsingETags: true #Are the conditional headers attached to ETag used to work out if to send a 304 Cache Redirect + diff --git a/web/page-handler.go b/web/page-handler.go index 00fc7ec..1589e79 100644 --- a/web/page-handler.go +++ b/web/page-handler.go @@ -2,23 +2,33 @@ package web import ( _ "embed" + "golang.captainalm.com/GOPackageHeaderServer/conf" "golang.captainalm.com/GOPackageHeaderServer/outputMeta" + "golang.captainalm.com/GOPackageHeaderServer/web/utils" "html/template" + "io" + "mime/multipart" "net/http" + "net/textproto" "strconv" + "time" ) type PageHandler struct { - Name string - CSS string - OutputPage bool - MetaOutput *outputMeta.PackageMetaTagOutputter + Name string + CSS string + OutputPage bool + RangeSupported bool + CacheSettings conf.CacheSettingsYaml + MetaOutput *outputMeta.PackageMetaTagOutputter } +var startTime = time.Now() + //go:embed output-page.html var outputPage string -var pageTemplateFuncMap template.FuncMap = template.FuncMap{ +var pageTemplateFuncMap = template.FuncMap{ "isNotEmpty": func(stringIn string) bool { return stringIn != "" }, @@ -28,30 +38,60 @@ func (pgh *PageHandler) ServeHTTP(writer http.ResponseWriter, request *http.Requ if request.Method == http.MethodGet || request.Method == http.MethodHead { tmpl, err := template.New("page-handler").Funcs(pageTemplateFuncMap).Parse(outputPage) if err != nil { - writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure") + utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Parsing Failure") return } tm := handlerTemplateMarshal{ PageHandler: *pgh, RequestPath: request.URL.Path, } - theBuffer := &BufferedWriter{} + theBuffer := &utils.BufferedWriter{} err = tmpl.Execute(theBuffer, tm) if err != nil { - writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure") + utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusInternalServerError, "Page Template Execution Failure") return } writer.Header().Set("Content-Length", strconv.Itoa(len(theBuffer.Data))) writer.Header().Set("Content-Type", "text/html; charset=utf-8") - if writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "") { - _, _ = writer.Write(theBuffer.Data) + utils.SetLastModifiedHeader(writer.Header(), startTime) + utils.SetCacheHeaderWithAge(writer.Header(), pgh.CacheSettings.MaxAge, startTime) + theETag := utils.GetValueForETagUsingBufferedWriter(theBuffer) + writer.Header().Set("ETag", theETag) + if utils.ProcessSupportedPreconditionsForNext(writer, request, startTime, theETag, pgh.CacheSettings.NotModifiedResponseUsingLastModified, pgh.CacheSettings.NotModifiedResponseUsingETags) { + httpRangeParts := utils.ProcessRangePreconditions(int64(len(theBuffer.Data)), writer, request, startTime, theETag, pgh.RangeSupported) + if httpRangeParts != nil { + if len(httpRangeParts) <= 1 { + var theWriter io.Writer = writer + if len(httpRangeParts) == 1 { + theWriter = utils.NewPartialRangeWriter(theWriter, httpRangeParts[0]) + } + _, _ = theWriter.Write(theBuffer.Data) + } else { + multWriter := multipart.NewWriter(writer) + writer.Header().Set("Content-Type", "multipart/byteranges; boundary="+multWriter.Boundary()) + for _, currentPart := range httpRangeParts { + mimePart, err := multWriter.CreatePart(textproto.MIMEHeader{ + "Content-Range": {currentPart.ToField(int64(len(theBuffer.Data)))}, + "Content-Type": {"text/plain; charset=utf-8"}, + }) + if err != nil { + break + } + _, err = mimePart.Write(theBuffer.Data[currentPart.Start : currentPart.Start+currentPart.Length]) + if err != nil { + break + } + } + _ = multWriter.Close() + } + } } } else { writer.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead) if request.Method == http.MethodOptions { - writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "") + utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusOK, "") } else { - writeResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "") + utils.WriteResponseHeaderCanWriteBody(request.Method, writer, http.StatusMethodNotAllowed, "") } } } diff --git a/web/utils.go b/web/utils.go deleted file mode 100644 index e5ffca5..0000000 --- a/web/utils.go +++ /dev/null @@ -1,33 +0,0 @@ -package web - -import ( - "net/http" - "strconv" -) - -func writeResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool { - hasBody := method != http.MethodHead && method != http.MethodOptions - if hasBody && message != "" { - rw.Header().Set("Content-Type", "text/plain; charset=utf-8") - rw.Header().Set("X-Content-Type-Options", "nosniff") - rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2)) - } - rw.WriteHeader(statusCode) - if hasBody { - if message != "" { - _, _ = rw.Write([]byte(message + "\r\n")) - return false - } - return true - } - return false -} - -type BufferedWriter struct { - Data []byte -} - -func (c *BufferedWriter) Write(p []byte) (n int, err error) { - c.Data = append(c.Data, p...) - return len(p), nil -} diff --git a/web/utils/buffered-writer.go b/web/utils/buffered-writer.go new file mode 100644 index 0000000..d3f50de --- /dev/null +++ b/web/utils/buffered-writer.go @@ -0,0 +1,23 @@ +package utils + +import ( + "crypto" + "encoding/hex" +) + +type BufferedWriter struct { + Data []byte +} + +func (c *BufferedWriter) Write(p []byte) (n int, err error) { + c.Data = append(c.Data, p...) + return len(p), nil +} + +func (c *BufferedWriter) GetHashString() string { + theHash := crypto.SHA1.New() + _, _ = theHash.Write(c.Data) + theSum := theHash.Sum(nil) + theHash.Reset() + return hex.EncodeToString(theSum) +} diff --git a/web/utils/content-range-value.go b/web/utils/content-range-value.go new file mode 100644 index 0000000..b0944b8 --- /dev/null +++ b/web/utils/content-range-value.go @@ -0,0 +1,78 @@ +package utils + +import ( + "strconv" + "strings" +) + +type ContentRangeValue struct { + Start, Length int64 +} + +func (rstrc ContentRangeValue) ToField(maxLength int64) string { + return "bytes " + strconv.FormatInt(rstrc.Start, 10) + "-" + strconv.FormatInt(rstrc.Start+rstrc.Length-1, 10) + "/" + strconv.FormatInt(maxLength, 10) +} + +func GetRanges(rangeStringIn string, maxLength int64) []ContentRangeValue { + actualRangeString := strings.TrimPrefix(rangeStringIn, "bytes=") + if strings.ContainsAny(actualRangeString, ",") { + seperated := strings.Split(actualRangeString, ",") + toReturn := make([]ContentRangeValue, len(seperated)) + pos := 0 + for _, s := range seperated { + if cRange, ok := GetRange(s, maxLength); ok { + toReturn[pos] = cRange + pos += 1 + } + } + if pos == 0 { + return nil + } + return toReturn[:pos] + } + if cRange, ok := GetRange(actualRangeString, maxLength); ok { + return []ContentRangeValue{cRange} + } + return nil +} + +func GetRange(rangePartIn string, maxLength int64) (ContentRangeValue, bool) { + before, after, done := strings.Cut(rangePartIn, "-") + before = strings.Trim(before, " ") + after = strings.Trim(after, " ") + if !done { + return ContentRangeValue{}, false + } + var parsedAfter, parsedBefore int64 = -1, -1 + if after != "" { + if parsed, err := strconv.ParseInt(after, 10, 64); err == nil { + parsedAfter = parsed + } else { + return ContentRangeValue{}, false + } + } + if before != "" { + if parsed, err := strconv.ParseInt(before, 10, 64); err == nil { + parsedBefore = parsed + } else { + return ContentRangeValue{}, false + } + } + if parsedBefore >= 0 && parsedAfter > parsedBefore && parsedAfter < maxLength { + return ContentRangeValue{ + Start: parsedBefore, + Length: parsedAfter - parsedBefore + 1, + }, true + } else if parsedAfter < 0 && parsedBefore >= 0 && parsedBefore < maxLength { + return ContentRangeValue{ + Start: parsedBefore, + Length: maxLength - parsedBefore, + }, true + } else if parsedBefore < 0 && parsedAfter >= 1 && maxLength-parsedAfter >= 0 { + return ContentRangeValue{ + Start: maxLength - parsedAfter, + Length: parsedAfter, + }, true + } + return ContentRangeValue{}, false +} diff --git a/web/utils/counting-writer.go b/web/utils/counting-writer.go new file mode 100644 index 0000000..67e8058 --- /dev/null +++ b/web/utils/counting-writer.go @@ -0,0 +1,10 @@ +package utils + +type CountingWriter struct { + Length int64 +} + +func (c *CountingWriter) Write(p []byte) (n int, err error) { + c.Length += int64(len(p)) + return len(p), nil +} diff --git a/web/utils/etag.go b/web/utils/etag.go new file mode 100644 index 0000000..fb2bb83 --- /dev/null +++ b/web/utils/etag.go @@ -0,0 +1,42 @@ +package utils + +import ( + "strings" +) + +func GetValueForETagUsingBufferedWriter(bWriter *BufferedWriter) string { + return "\"" + bWriter.GetHashString() + "\"" +} + +func GetETagValues(stringIn string) []string { + if strings.ContainsAny(stringIn, ",") { + seperated := strings.Split(stringIn, ",") + toReturn := make([]string, len(seperated)) + pos := 0 + for _, s := range seperated { + cETag := GetETagValue(s) + if cETag != "" { + toReturn[pos] = cETag + pos += 1 + } + } + if pos == 0 { + return nil + } + return toReturn[:pos] + } + toReturn := []string{GetETagValue(stringIn)} + if toReturn[0] == "" { + return nil + } + return toReturn +} + +func GetETagValue(stringIn string) string { + startIndex := strings.IndexAny(stringIn, "\"") + 1 + endIndex := strings.LastIndexAny(stringIn, "\"") + if endIndex > startIndex { + return stringIn[startIndex:endIndex] + } + return "" +} diff --git a/web/utils/partial-range-writer.go b/web/utils/partial-range-writer.go new file mode 100644 index 0000000..2f82753 --- /dev/null +++ b/web/utils/partial-range-writer.go @@ -0,0 +1,47 @@ +package utils + +import "io" + +func NewPartialRangeWriter(writerIn io.Writer, httpRangeIn ContentRangeValue) io.Writer { + return &PartialRangeWriter{ + passedWriter: writerIn, + passedWriterIndex: 0, + httpRange: httpRangeIn, + exclusiveLastIndex: httpRangeIn.Start + httpRangeIn.Length, + } +} + +type PartialRangeWriter struct { + passedWriter io.Writer + passedWriterIndex int64 + exclusiveLastIndex int64 + httpRange ContentRangeValue +} + +func (prw *PartialRangeWriter) Write(p []byte) (n int, err error) { + var pOffsetIndex int64 = -1 + if prw.passedWriterIndex >= prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex { + pOffsetIndex = 0 + } else if prw.passedWriterIndex+int64(len(p)) > prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex { + pOffsetIndex = prw.httpRange.Start - prw.passedWriterIndex + prw.passedWriterIndex += pOffsetIndex + } else { + prw.passedWriterIndex += int64(len(p)) + } + if pOffsetIndex >= 0 { + if prw.passedWriterIndex+(int64(len(p))-pOffsetIndex) <= prw.exclusiveLastIndex { + written, err := prw.passedWriter.Write(p[pOffsetIndex:]) + prw.passedWriterIndex += int64(written) + if err != nil { + return written, err + } + } else { + written, err := prw.passedWriter.Write(p[pOffsetIndex : prw.exclusiveLastIndex-prw.passedWriterIndex+pOffsetIndex]) + prw.passedWriterIndex += int64(written) + if err != nil { + return written, err + } + } + } + return n, nil +} diff --git a/web/utils/process-preconditions.go b/web/utils/process-preconditions.go new file mode 100644 index 0000000..71b2810 --- /dev/null +++ b/web/utils/process-preconditions.go @@ -0,0 +1,146 @@ +package utils + +import ( + "mime/multipart" + "net/http" + "net/textproto" + "strconv" + "strings" + "time" +) + +func ProcessSupportedPreconditionsForNext(rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, noBypassModify bool, noBypassMatch bool) bool { + theStrippedETag := GetETagValue(etag) + if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-None-Match") != "" { + etagVals := GetETagValues(req.Header.Get("If-None-Match")) + conditionSuccess := false + for _, s := range etagVals { + if s == theStrippedETag { + conditionSuccess = true + break + } + } + if conditionSuccess { + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "") + return false + } + } + + if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-Match") != "" { + etagVals := GetETagValues(req.Header.Get("If-Match")) + conditionFailed := true + for _, s := range etagVals { + if s == theStrippedETag { + conditionFailed = false + break + } + } + if conditionFailed { + SwitchToNonCachingHeaders(rw.Header()) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "") + return false + } + } + + if noBypassModify && !modT.IsZero() && req.Header.Get("If-Modified-Since") != "" { + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Modified-Since")) + if err == nil && modT.Before(parse) || strings.EqualFold(modT.Format(http.TimeFormat), req.Header.Get("If-Modified-Since")) { + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "") + return false + } + } + + if noBypassModify && !modT.IsZero() && req.Header.Get("If-Unmodified-Since") != "" { + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Unmodified-Since")) + if err == nil && modT.After(parse) { + SwitchToNonCachingHeaders(rw.Header()) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "") + return false + } + } + + return true +} + +func ProcessRangePreconditions(maxLength int64, rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, supported bool) []ContentRangeValue { + canDoRange := supported + theStrippedETag := GetETagValue(etag) + modTStr := modT.Format(http.TimeFormat) + + if canDoRange { + rw.Header().Set("Accept-Ranges", "bytes") + } + + if canDoRange && !modT.IsZero() && strings.HasSuffix(req.Header.Get("If-Range"), "GMT") { + newModT, err := time.Parse(http.TimeFormat, modTStr) + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Range")) + if err == nil && !newModT.Equal(parse) { + canDoRange = false + } + } else if canDoRange && theStrippedETag != "" && req.Header.Get("If-Range") != "" { + if GetETagValue(req.Header.Get("If-Range")) != theStrippedETag { + canDoRange = false + } + } + + if canDoRange && strings.HasPrefix(req.Header.Get("Range"), "bytes=") { + if theRanges := GetRanges(req.Header.Get("Range"), maxLength); len(theRanges) != 0 { + if len(theRanges) == 1 { + rw.Header().Set("Content-Length", strconv.FormatInt(theRanges[0].Length, 10)) + rw.Header().Set("Content-Range", theRanges[0].ToField(maxLength)) + } else { + theSize := GetMultipartLength(theRanges, rw.Header().Get("Content-Type"), maxLength) + rw.Header().Set("Content-Length", strconv.FormatInt(theSize, 10)) + } + if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPartialContent, "") { + return theRanges + } else { + return nil + } + } else { + SwitchToNonCachingHeaders(rw.Header()) + rw.Header().Set("Content-Range", "bytes */"+strconv.FormatInt(maxLength, 10)) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusRequestedRangeNotSatisfiable, "") + return nil + } + } + if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") { + return make([]ContentRangeValue, 0) + } + return nil +} + +func GetMultipartLength(parts []ContentRangeValue, contentType string, maxLength int64) int64 { + cWriter := &CountingWriter{Length: 0} + var returnLength int64 = 0 + multWriter := multipart.NewWriter(cWriter) + for _, currentPart := range parts { + _, _ = multWriter.CreatePart(textproto.MIMEHeader{ + "Content-Range": {currentPart.ToField(maxLength)}, + "Content-Type": {contentType}, + }) + returnLength += currentPart.Length + } + _ = multWriter.Close() + returnLength += cWriter.Length + return returnLength +} + +func WriteResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool { + hasBody := method != http.MethodHead && method != http.MethodOptions + if hasBody && message != "" { + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + rw.Header().Set("X-Content-Type-Options", "nosniff") + rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2)) + SetNeverCacheHeader(rw.Header()) + } + rw.WriteHeader(statusCode) + if hasBody { + if message != "" { + _, _ = rw.Write([]byte(message + "\r\n")) + return false + } + return true + } + return false +} diff --git a/web/utils/utils.go b/web/utils/utils.go new file mode 100644 index 0000000..551caa4 --- /dev/null +++ b/web/utils/utils.go @@ -0,0 +1,45 @@ +package utils + +import ( + "net/http" + "strconv" + "time" +) + +func SetNeverCacheHeader(header http.Header) { + header.Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + header.Set("Pragma", "no-cache") +} + +func SetLastModifiedHeader(header http.Header, modTime time.Time) { + if !modTime.IsZero() { + header.Set("Last-Modified", modTime.UTC().Format(http.TimeFormat)) + } +} + +func SetCacheHeaderWithAge(header http.Header, maxAge uint, modifiedTime time.Time) { + header.Set("Cache-Control", "max-age="+strconv.Itoa(int(maxAge))+", must-revalidate") + if maxAge > 0 { + checkerSecondsBetween := int64(time.Now().UTC().Sub(modifiedTime.UTC()).Seconds()) + if checkerSecondsBetween < 0 { + checkerSecondsBetween *= -1 + } + header.Set("Age", strconv.FormatUint(uint64(checkerSecondsBetween)%uint64(maxAge), 10)) + } +} + +func SwitchToNonCachingHeaders(header http.Header) { + SetNeverCacheHeader(header) + if header.Get("Last-Modified") != "" { + header.Del("Last-Modified") + } + if header.Get("Age") != "" { + header.Del("Age") + } + if header.Get("Expires") != "" { + header.Del("Expires") + } + if header.Get("ETag") != "" { + header.Del("ETag") + } +} diff --git a/web/web.go b/web/web.go index 94ba46c..ba58769 100644 --- a/web/web.go +++ b/web/web.go @@ -3,6 +3,7 @@ package web import ( "github.com/gorilla/mux" "golang.captainalm.com/GOPackageHeaderServer/conf" + "golang.captainalm.com/GOPackageHeaderServer/web/utils" "log" "net/http" "strings" @@ -13,10 +14,12 @@ func New(yaml conf.ConfigYaml) (*http.Server, map[string]*PageHandler) { var pages = make(map[string]*PageHandler) for _, zc := range yaml.Zones { currentPage := &PageHandler{ - Name: zc.Name, - CSS: zc.CssURL, - OutputPage: zc.HavePageContents, - MetaOutput: zc.GetPackageMetaTagOutputter(), + Name: zc.Name, + CSS: zc.CssURL, + OutputPage: zc.HavePageContents, + RangeSupported: zc.RangeSupported, + MetaOutput: zc.GetPackageMetaTagOutputter(), + CacheSettings: zc.CacheSettings, } for _, d := range zc.Domains { ld := strings.ToLower(d) @@ -56,13 +59,13 @@ func runBackgroundHttp(s *http.Server) { func domainNotAllowed(rw http.ResponseWriter, req *http.Request) { if req.Method == http.MethodGet || req.Method == http.MethodHead { - writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed") + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed") } else { rw.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead) if req.Method == http.MethodOptions { - writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") } else { - writeResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "") + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "") } } }