From 11deb796e82ea107fd0d57d3b9af1a3c3a51f0f0 Mon Sep 17 00:00:00 2001 From: Captain ALM Date: Thu, 14 Jul 2022 23:25:43 +0100 Subject: [PATCH] Begin adding page handling system, add utils for the pageHandler system. --- pageHandler/get-router.go | 42 ++++++ pageHandler/page-handler.go | 88 +++++++++++-- pageHandler/page-provider.go | 7 + pageHandler/pages.go | 11 ++ pageHandler/utils/buffered-writer.go | 23 ++++ pageHandler/utils/content-range-value.go | 78 +++++++++++ pageHandler/utils/counting-writer.go | 10 ++ pageHandler/utils/etag.go | 42 ++++++ pageHandler/utils/partial-range-writer.go | 47 +++++++ pageHandler/utils/process-preconditions.go | 146 +++++++++++++++++++++ pageHandler/utils/utils.go | 45 +++++++ 11 files changed, 530 insertions(+), 9 deletions(-) create mode 100644 pageHandler/get-router.go create mode 100644 pageHandler/page-provider.go create mode 100644 pageHandler/pages.go create mode 100644 pageHandler/utils/buffered-writer.go create mode 100644 pageHandler/utils/content-range-value.go create mode 100644 pageHandler/utils/counting-writer.go create mode 100644 pageHandler/utils/etag.go create mode 100644 pageHandler/utils/partial-range-writer.go create mode 100644 pageHandler/utils/process-preconditions.go create mode 100644 pageHandler/utils/utils.go diff --git a/pageHandler/get-router.go b/pageHandler/get-router.go new file mode 100644 index 0000000..804b5f5 --- /dev/null +++ b/pageHandler/get-router.go @@ -0,0 +1,42 @@ +package pageHandler + +import ( + "github.com/gorilla/mux" + "golang.captainalm.com/cityuni-webserver/conf" + "golang.captainalm.com/cityuni-webserver/pageHandler/utils" + "net/http" +) + +var theRouter *mux.Router +var thePageHandler *PageHandler + +func GetRouter(config conf.ConfigYaml) http.Handler { + if theRouter == nil { + theRouter = mux.NewRouter() + if thePageHandler == nil { + thePageHandler = NewPageHandler(config.Serve) + } + if len(config.Serve.Domains) == 0 { + theRouter.PathPrefix("/").HandlerFunc(thePageHandler.ServeHTTP) + } else { + for _, domain := range config.Serve.Domains { + theRouter.Host(domain).HandlerFunc(thePageHandler.ServeHTTP) + } + theRouter.PathPrefix("/").HandlerFunc(domainNotAllowed) + } + } + return theRouter +} + +func domainNotAllowed(rw http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodGet || req.Method == http.MethodHead { + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotFound, "Domain Not Allowed") + } else { + rw.Header().Set("Allow", http.MethodOptions+", "+http.MethodGet+", "+http.MethodHead) + if req.Method == http.MethodOptions { + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") + } else { + utils.WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusMethodNotAllowed, "") + } + } +} diff --git a/pageHandler/page-handler.go b/pageHandler/page-handler.go index 61e19b6..7d311be 100644 --- a/pageHandler/page-handler.go +++ b/pageHandler/page-handler.go @@ -1,17 +1,87 @@ package pageHandler import ( - "github.com/gorilla/mux" "golang.captainalm.com/cityuni-webserver/conf" "net/http" + "strings" + "sync" ) -var theRouter *mux.Router - -func GetRouter(config conf.ConfigYaml) http.Handler { - if theRouter == nil { - theRouter = mux.NewRouter() - //Mux routing stuff - } - return theRouter +type PageHandler struct { + PageContentsCache map[string][]byte + PageProviders map[string]PageProvider + pageContentsCacheRWMutex *sync.RWMutex + RangeSupported bool + CacheSettings conf.CacheSettingsYaml +} + +func NewPageHandler(config conf.ServeYaml) *PageHandler { + var thePCCMap map[string][]byte + var theMutex *sync.RWMutex + if config.CacheSettings.EnableContentsCaching { + thePCCMap = make(map[string][]byte) + theMutex = &sync.RWMutex{} + } + return &PageHandler{ + PageContentsCache: thePCCMap, + PageProviders: GetProviders(config.CacheSettings.EnableTemplateCaching), + pageContentsCacheRWMutex: theMutex, + RangeSupported: config.RangeSupported, + CacheSettings: config.CacheSettings, + } +} + +func (ph *PageHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + //Provide processing for requests using providers +} + +func (ph *PageHandler) PurgeContentsCache(path string, query string) { + if ph.CacheSettings.EnableContentsCaching { + if path == "" { + ph.pageContentsCacheRWMutex.Lock() + ph.PageContentsCache = make(map[string][]byte) + ph.pageContentsCacheRWMutex.Unlock() + } else { + if strings.HasSuffix(path, "/") { + ph.pageContentsCacheRWMutex.RLock() + toDelete := make([]string, len(ph.PageContentsCache)) + theSize := 0 + for cPath := range ph.PageContentsCache { + dPath := strings.Split(cPath, "?")[0] + if dPath == path || dPath == path[:len(path)-1] { + toDelete[theSize] = cPath + theSize++ + } + } + ph.pageContentsCacheRWMutex.RUnlock() + ph.pageContentsCacheRWMutex.Lock() + for i := 0; i < theSize; i++ { + delete(ph.PageContentsCache, toDelete[i]) + } + ph.pageContentsCacheRWMutex.Unlock() + } else { + ph.pageContentsCacheRWMutex.Lock() + if query == "" { + delete(ph.PageContentsCache, path) + } else { + delete(ph.PageContentsCache, path+"?"+query) + } + ph.pageContentsCacheRWMutex.Unlock() + } + } + } +} + +func (ph *PageHandler) PurgeTemplateCache(path string) { + if ph.CacheSettings.EnableTemplateCaching && ph.CacheSettings.EnableTemplateCachePurge { + if path == "" { + for _, pageProvider := range ph.PageProviders { + pageProvider.PurgeTemplate() + } + } else { + if pageProvider, ok := ph.PageProviders[path]; ok { + pageProvider.PurgeTemplate() + } + } + } } diff --git a/pageHandler/page-provider.go b/pageHandler/page-provider.go new file mode 100644 index 0000000..d8ebb44 --- /dev/null +++ b/pageHandler/page-provider.go @@ -0,0 +1,7 @@ +package pageHandler + +type PageProvider interface { + GetPath() string + GetContents(urlParameters map[string]string) (contentType string, contents []byte) + PurgeTemplate() +} diff --git a/pageHandler/pages.go b/pageHandler/pages.go new file mode 100644 index 0000000..32bd7b6 --- /dev/null +++ b/pageHandler/pages.go @@ -0,0 +1,11 @@ +package pageHandler + +var providers map[string]PageProvider + +func GetProviders(cacheTemplates bool) map[string]PageProvider { + if providers == nil { + providers = make(map[string]PageProvider) + //Add the providers in the pages sub package + } + return providers +} diff --git a/pageHandler/utils/buffered-writer.go b/pageHandler/utils/buffered-writer.go new file mode 100644 index 0000000..d3f50de --- /dev/null +++ b/pageHandler/utils/buffered-writer.go @@ -0,0 +1,23 @@ +package utils + +import ( + "crypto" + "encoding/hex" +) + +type BufferedWriter struct { + Data []byte +} + +func (c *BufferedWriter) Write(p []byte) (n int, err error) { + c.Data = append(c.Data, p...) + return len(p), nil +} + +func (c *BufferedWriter) GetHashString() string { + theHash := crypto.SHA1.New() + _, _ = theHash.Write(c.Data) + theSum := theHash.Sum(nil) + theHash.Reset() + return hex.EncodeToString(theSum) +} diff --git a/pageHandler/utils/content-range-value.go b/pageHandler/utils/content-range-value.go new file mode 100644 index 0000000..b0944b8 --- /dev/null +++ b/pageHandler/utils/content-range-value.go @@ -0,0 +1,78 @@ +package utils + +import ( + "strconv" + "strings" +) + +type ContentRangeValue struct { + Start, Length int64 +} + +func (rstrc ContentRangeValue) ToField(maxLength int64) string { + return "bytes " + strconv.FormatInt(rstrc.Start, 10) + "-" + strconv.FormatInt(rstrc.Start+rstrc.Length-1, 10) + "/" + strconv.FormatInt(maxLength, 10) +} + +func GetRanges(rangeStringIn string, maxLength int64) []ContentRangeValue { + actualRangeString := strings.TrimPrefix(rangeStringIn, "bytes=") + if strings.ContainsAny(actualRangeString, ",") { + seperated := strings.Split(actualRangeString, ",") + toReturn := make([]ContentRangeValue, len(seperated)) + pos := 0 + for _, s := range seperated { + if cRange, ok := GetRange(s, maxLength); ok { + toReturn[pos] = cRange + pos += 1 + } + } + if pos == 0 { + return nil + } + return toReturn[:pos] + } + if cRange, ok := GetRange(actualRangeString, maxLength); ok { + return []ContentRangeValue{cRange} + } + return nil +} + +func GetRange(rangePartIn string, maxLength int64) (ContentRangeValue, bool) { + before, after, done := strings.Cut(rangePartIn, "-") + before = strings.Trim(before, " ") + after = strings.Trim(after, " ") + if !done { + return ContentRangeValue{}, false + } + var parsedAfter, parsedBefore int64 = -1, -1 + if after != "" { + if parsed, err := strconv.ParseInt(after, 10, 64); err == nil { + parsedAfter = parsed + } else { + return ContentRangeValue{}, false + } + } + if before != "" { + if parsed, err := strconv.ParseInt(before, 10, 64); err == nil { + parsedBefore = parsed + } else { + return ContentRangeValue{}, false + } + } + if parsedBefore >= 0 && parsedAfter > parsedBefore && parsedAfter < maxLength { + return ContentRangeValue{ + Start: parsedBefore, + Length: parsedAfter - parsedBefore + 1, + }, true + } else if parsedAfter < 0 && parsedBefore >= 0 && parsedBefore < maxLength { + return ContentRangeValue{ + Start: parsedBefore, + Length: maxLength - parsedBefore, + }, true + } else if parsedBefore < 0 && parsedAfter >= 1 && maxLength-parsedAfter >= 0 { + return ContentRangeValue{ + Start: maxLength - parsedAfter, + Length: parsedAfter, + }, true + } + return ContentRangeValue{}, false +} diff --git a/pageHandler/utils/counting-writer.go b/pageHandler/utils/counting-writer.go new file mode 100644 index 0000000..67e8058 --- /dev/null +++ b/pageHandler/utils/counting-writer.go @@ -0,0 +1,10 @@ +package utils + +type CountingWriter struct { + Length int64 +} + +func (c *CountingWriter) Write(p []byte) (n int, err error) { + c.Length += int64(len(p)) + return len(p), nil +} diff --git a/pageHandler/utils/etag.go b/pageHandler/utils/etag.go new file mode 100644 index 0000000..fb2bb83 --- /dev/null +++ b/pageHandler/utils/etag.go @@ -0,0 +1,42 @@ +package utils + +import ( + "strings" +) + +func GetValueForETagUsingBufferedWriter(bWriter *BufferedWriter) string { + return "\"" + bWriter.GetHashString() + "\"" +} + +func GetETagValues(stringIn string) []string { + if strings.ContainsAny(stringIn, ",") { + seperated := strings.Split(stringIn, ",") + toReturn := make([]string, len(seperated)) + pos := 0 + for _, s := range seperated { + cETag := GetETagValue(s) + if cETag != "" { + toReturn[pos] = cETag + pos += 1 + } + } + if pos == 0 { + return nil + } + return toReturn[:pos] + } + toReturn := []string{GetETagValue(stringIn)} + if toReturn[0] == "" { + return nil + } + return toReturn +} + +func GetETagValue(stringIn string) string { + startIndex := strings.IndexAny(stringIn, "\"") + 1 + endIndex := strings.LastIndexAny(stringIn, "\"") + if endIndex > startIndex { + return stringIn[startIndex:endIndex] + } + return "" +} diff --git a/pageHandler/utils/partial-range-writer.go b/pageHandler/utils/partial-range-writer.go new file mode 100644 index 0000000..2f82753 --- /dev/null +++ b/pageHandler/utils/partial-range-writer.go @@ -0,0 +1,47 @@ +package utils + +import "io" + +func NewPartialRangeWriter(writerIn io.Writer, httpRangeIn ContentRangeValue) io.Writer { + return &PartialRangeWriter{ + passedWriter: writerIn, + passedWriterIndex: 0, + httpRange: httpRangeIn, + exclusiveLastIndex: httpRangeIn.Start + httpRangeIn.Length, + } +} + +type PartialRangeWriter struct { + passedWriter io.Writer + passedWriterIndex int64 + exclusiveLastIndex int64 + httpRange ContentRangeValue +} + +func (prw *PartialRangeWriter) Write(p []byte) (n int, err error) { + var pOffsetIndex int64 = -1 + if prw.passedWriterIndex >= prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex { + pOffsetIndex = 0 + } else if prw.passedWriterIndex+int64(len(p)) > prw.httpRange.Start && prw.passedWriterIndex < prw.exclusiveLastIndex { + pOffsetIndex = prw.httpRange.Start - prw.passedWriterIndex + prw.passedWriterIndex += pOffsetIndex + } else { + prw.passedWriterIndex += int64(len(p)) + } + if pOffsetIndex >= 0 { + if prw.passedWriterIndex+(int64(len(p))-pOffsetIndex) <= prw.exclusiveLastIndex { + written, err := prw.passedWriter.Write(p[pOffsetIndex:]) + prw.passedWriterIndex += int64(written) + if err != nil { + return written, err + } + } else { + written, err := prw.passedWriter.Write(p[pOffsetIndex : prw.exclusiveLastIndex-prw.passedWriterIndex+pOffsetIndex]) + prw.passedWriterIndex += int64(written) + if err != nil { + return written, err + } + } + } + return n, nil +} diff --git a/pageHandler/utils/process-preconditions.go b/pageHandler/utils/process-preconditions.go new file mode 100644 index 0000000..71b2810 --- /dev/null +++ b/pageHandler/utils/process-preconditions.go @@ -0,0 +1,146 @@ +package utils + +import ( + "mime/multipart" + "net/http" + "net/textproto" + "strconv" + "strings" + "time" +) + +func ProcessSupportedPreconditionsForNext(rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, noBypassModify bool, noBypassMatch bool) bool { + theStrippedETag := GetETagValue(etag) + if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-None-Match") != "" { + etagVals := GetETagValues(req.Header.Get("If-None-Match")) + conditionSuccess := false + for _, s := range etagVals { + if s == theStrippedETag { + conditionSuccess = true + break + } + } + if conditionSuccess { + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "") + return false + } + } + + if noBypassMatch && theStrippedETag != "" && req.Header.Get("If-Match") != "" { + etagVals := GetETagValues(req.Header.Get("If-Match")) + conditionFailed := true + for _, s := range etagVals { + if s == theStrippedETag { + conditionFailed = false + break + } + } + if conditionFailed { + SwitchToNonCachingHeaders(rw.Header()) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "") + return false + } + } + + if noBypassModify && !modT.IsZero() && req.Header.Get("If-Modified-Since") != "" { + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Modified-Since")) + if err == nil && modT.Before(parse) || strings.EqualFold(modT.Format(http.TimeFormat), req.Header.Get("If-Modified-Since")) { + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusNotModified, "") + return false + } + } + + if noBypassModify && !modT.IsZero() && req.Header.Get("If-Unmodified-Since") != "" { + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Unmodified-Since")) + if err == nil && modT.After(parse) { + SwitchToNonCachingHeaders(rw.Header()) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPreconditionFailed, "") + return false + } + } + + return true +} + +func ProcessRangePreconditions(maxLength int64, rw http.ResponseWriter, req *http.Request, modT time.Time, etag string, supported bool) []ContentRangeValue { + canDoRange := supported + theStrippedETag := GetETagValue(etag) + modTStr := modT.Format(http.TimeFormat) + + if canDoRange { + rw.Header().Set("Accept-Ranges", "bytes") + } + + if canDoRange && !modT.IsZero() && strings.HasSuffix(req.Header.Get("If-Range"), "GMT") { + newModT, err := time.Parse(http.TimeFormat, modTStr) + parse, err := time.Parse(http.TimeFormat, req.Header.Get("If-Range")) + if err == nil && !newModT.Equal(parse) { + canDoRange = false + } + } else if canDoRange && theStrippedETag != "" && req.Header.Get("If-Range") != "" { + if GetETagValue(req.Header.Get("If-Range")) != theStrippedETag { + canDoRange = false + } + } + + if canDoRange && strings.HasPrefix(req.Header.Get("Range"), "bytes=") { + if theRanges := GetRanges(req.Header.Get("Range"), maxLength); len(theRanges) != 0 { + if len(theRanges) == 1 { + rw.Header().Set("Content-Length", strconv.FormatInt(theRanges[0].Length, 10)) + rw.Header().Set("Content-Range", theRanges[0].ToField(maxLength)) + } else { + theSize := GetMultipartLength(theRanges, rw.Header().Get("Content-Type"), maxLength) + rw.Header().Set("Content-Length", strconv.FormatInt(theSize, 10)) + } + if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusPartialContent, "") { + return theRanges + } else { + return nil + } + } else { + SwitchToNonCachingHeaders(rw.Header()) + rw.Header().Set("Content-Range", "bytes */"+strconv.FormatInt(maxLength, 10)) + WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusRequestedRangeNotSatisfiable, "") + return nil + } + } + if WriteResponseHeaderCanWriteBody(req.Method, rw, http.StatusOK, "") { + return make([]ContentRangeValue, 0) + } + return nil +} + +func GetMultipartLength(parts []ContentRangeValue, contentType string, maxLength int64) int64 { + cWriter := &CountingWriter{Length: 0} + var returnLength int64 = 0 + multWriter := multipart.NewWriter(cWriter) + for _, currentPart := range parts { + _, _ = multWriter.CreatePart(textproto.MIMEHeader{ + "Content-Range": {currentPart.ToField(maxLength)}, + "Content-Type": {contentType}, + }) + returnLength += currentPart.Length + } + _ = multWriter.Close() + returnLength += cWriter.Length + return returnLength +} + +func WriteResponseHeaderCanWriteBody(method string, rw http.ResponseWriter, statusCode int, message string) bool { + hasBody := method != http.MethodHead && method != http.MethodOptions + if hasBody && message != "" { + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + rw.Header().Set("X-Content-Type-Options", "nosniff") + rw.Header().Set("Content-Length", strconv.Itoa(len(message)+2)) + SetNeverCacheHeader(rw.Header()) + } + rw.WriteHeader(statusCode) + if hasBody { + if message != "" { + _, _ = rw.Write([]byte(message + "\r\n")) + return false + } + return true + } + return false +} diff --git a/pageHandler/utils/utils.go b/pageHandler/utils/utils.go new file mode 100644 index 0000000..551caa4 --- /dev/null +++ b/pageHandler/utils/utils.go @@ -0,0 +1,45 @@ +package utils + +import ( + "net/http" + "strconv" + "time" +) + +func SetNeverCacheHeader(header http.Header) { + header.Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + header.Set("Pragma", "no-cache") +} + +func SetLastModifiedHeader(header http.Header, modTime time.Time) { + if !modTime.IsZero() { + header.Set("Last-Modified", modTime.UTC().Format(http.TimeFormat)) + } +} + +func SetCacheHeaderWithAge(header http.Header, maxAge uint, modifiedTime time.Time) { + header.Set("Cache-Control", "max-age="+strconv.Itoa(int(maxAge))+", must-revalidate") + if maxAge > 0 { + checkerSecondsBetween := int64(time.Now().UTC().Sub(modifiedTime.UTC()).Seconds()) + if checkerSecondsBetween < 0 { + checkerSecondsBetween *= -1 + } + header.Set("Age", strconv.FormatUint(uint64(checkerSecondsBetween)%uint64(maxAge), 10)) + } +} + +func SwitchToNonCachingHeaders(header http.Header) { + SetNeverCacheHeader(header) + if header.Get("Last-Modified") != "" { + header.Del("Last-Modified") + } + if header.Get("Age") != "" { + header.Del("Age") + } + if header.Get("Expires") != "" { + header.Del("Expires") + } + if header.Get("ETag") != "" { + header.Del("ETag") + } +}