pax_global_header 0000666 0000000 0000000 00000000064 13545335254 0014523 g ustar 00root root 0000000 0000000 52 comment=6ab1d606c1779bcd94392cae0bb4d7bc2c6360e5
gopher-parse-sitemap-0.1/ 0000775 0000000 0000000 00000000000 13545335254 0015417 5 ustar 00root root 0000000 0000000 gopher-parse-sitemap-0.1/.gitignore 0000664 0000000 0000000 00000000300 13545335254 0017400 0 ustar 00root root 0000000 0000000 # Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
gopher-parse-sitemap-0.1/.travis.yml 0000664 0000000 0000000 00000000202 13545335254 0017522 0 ustar 00root root 0000000 0000000 language: go
before_install:
- go get -t -v ./...
script:
- go test -race -coverprofile=coverage.txt -covermode=atomic gopher-parse-sitemap-0.1/LICENSE 0000664 0000000 0000000 00000002147 13545335254 0016430 0 ustar 00root root 0000000 0000000 MIT License
Copyright (c) 2019 Akulyakov Artem (akulyakov.artem@gmail.com, https://github.com/oxffaa)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
gopher-parse-sitemap-0.1/README.md 0000664 0000000 0000000 00000003241 13545335254 0016676 0 ustar 00root root 0000000 0000000 # gopher-parse-sitemap
A high effective library for parsing big sitemaps. See https://www.sitemaps.org/ for more information about the sitemap format.
## Why yet another sitemaps parsing library?
Time by time needs to parse really huge sitemaps. If you just unmarshal the whole file to an array of structures it produces high memory usage and the application can crash due to OOM (out of memory error).
The solution is to handle sitemap entries on the fly. That is read one entity, consume it, repeat while there are unhandled items in the sitemap.
```golang
err := sitemap.ParseFromFile("./testdata/sitemap.xml", func(e Entry) error {
return fmt.Println(e.GetLocation())
})
```
### I need parse only small and medium-sized sitemaps. Should I use this library?
Yes. Of course, you can just load a sitemap to memory.
```golang
result := make([]string, 0, 0)
err := ParseIndexFromFile("./testdata/sitemap-index.xml", func(e IndexEntry) error {
result = append(result, e.GetLocation())
return nil
})
```
But if you are pretty sure that you don't need to handle big-sized sitemaps, maybe better to choose the library with simpler and suitable API. In that case, you can try projects like https://github.com/yterajima/go-sitemap, https://github.com/snabb/sitemap, and https://github.com/decaseal/go-sitemap-parser.
## Install
Installation is pretty easy, just do:
```bash
go get -u github.com/oxffaa/gopher-parse-sitemap
```
After that import it:
```golang
import "github.com/oxffaa/gopher-parse-sitemap"
```
Well done, you can start to create something awesome.
## Documentation
Please, see here https://godoc.org/github.com/oxffaa/gopher-parse-sitemap for documentation.
gopher-parse-sitemap-0.1/go.mod 0000664 0000000 0000000 00000000067 13545335254 0016530 0 ustar 00root root 0000000 0000000 module github.com/oxffaa/gopher-parse-sitemap
go 1.13
gopher-parse-sitemap-0.1/sitemap.go 0000664 0000000 0000000 00000011461 13545335254 0017413 0 ustar 00root root 0000000 0000000 // Package sitemap provides primitives for high effective parsing of huge
// sitemap files.
package sitemap
import (
"encoding/xml"
"io"
"net/http"
"os"
"time"
)
// Frequency is a type alias for change frequency.
type Frequency = string
// Change frequency constants set describes how frequently a page is changed.
const (
Always Frequency = "always" // A page is changed always
Hourly Frequency = "hourly" // A page is changed every hour
Daily Frequency = "daily" // A page is changed every day
Weekly Frequency = "weekly" // A page is changed every week
Monthly Frequency = "monthly" // A page is changed every month
Yearly Frequency = "yearly" // A page is changed every year
Never Frequency = "never" // A page is changed never
)
// Entry is an interface describes an element \ an URL in the sitemap file.
// Keep in mind. It is implemented by a totally immutable entity so you should
// minimize calls count because it can produce additional memory allocations.
//
// GetLocation returns URL of the page.
// GetLocation must return a non-nil and not empty string value.
//
// GetLastModified parses and returns date and time of last modification of the page.
// GetLastModified can return nil or a valid time.Time instance.
// Be careful. Each call return new time.Time instance.
//
// GetChangeFrequency returns string value indicates how frequent the page is changed.
// GetChangeFrequency returns non-nil string value. See Frequency consts set.
//
// GetPriority return priority of the page.
// The valid value is between 0.0 and 1.0, the default value is 0.5.
//
// You shouldn't implement this interface in your types.
type Entry interface {
GetLocation() string
GetLastModified() *time.Time
GetChangeFrequency() Frequency
GetPriority() float32
}
// IndexEntry is an interface describes an element \ an URL in a sitemap index file.
// Keep in mind. It is implemented by a totally immutable entity so you should
// minimize calls count because it can produce additional memory allocations.
//
// GetLocation returns URL of a sitemap file.
// GetLocation must return a non-nil and not empty string value.
//
// GetLastModified parses and returns date and time of last modification of sitemap.
// GetLastModified can return nil or a valid time.Time instance.
// Be careful. Each call return new time.Time instance.
//
// You shouldn't implement this interface in your types.
type IndexEntry interface {
GetLocation() string
GetLastModified() *time.Time
}
// EntryConsumer is a type represents consumer of parsed sitemaps entries
type EntryConsumer func(Entry) error
// Parse parses data which provides by the reader and for each sitemap
// entry calls the consumer's function.
func Parse(reader io.Reader, consumer EntryConsumer) error {
return parseLoop(reader, func(d *xml.Decoder, se *xml.StartElement) error {
return entryParser(d, se, consumer)
})
}
// ParseFromFile reads sitemap from a file, parses it and for each sitemap
// entry calls the consumer's function.
func ParseFromFile(sitemapPath string, consumer EntryConsumer) error {
sitemapFile, err := os.OpenFile(sitemapPath, os.O_RDONLY, os.ModeExclusive)
if err != nil {
return err
}
defer sitemapFile.Close()
return Parse(sitemapFile, consumer)
}
// ParseFromSite downloads sitemap from a site, parses it and for each sitemap
// entry calls the consumer's function.
func ParseFromSite(url string, consumer EntryConsumer) error {
res, err := http.Get(url)
if err != nil {
return err
}
defer res.Body.Close()
return Parse(res.Body, consumer)
}
// IndexEntryConsumer is a type represents consumer of parsed sitemaps indexes entries
type IndexEntryConsumer func(IndexEntry) error
// ParseIndex parses data which provides by the reader and for each sitemap index
// entry calls the consumer's function.
func ParseIndex(reader io.Reader, consumer IndexEntryConsumer) error {
return parseLoop(reader, func(d *xml.Decoder, se *xml.StartElement) error {
return indexEntryParser(d, se, consumer)
})
}
// ParseIndexFromFile reads sitemap index from a file, parses it and for each sitemap
// index entry calls the consumer's function.
func ParseIndexFromFile(sitemapPath string, consumer IndexEntryConsumer) error {
sitemapFile, err := os.OpenFile(sitemapPath, os.O_RDONLY, os.ModeExclusive)
if err != nil {
return err
}
defer sitemapFile.Close()
return ParseIndex(sitemapFile, consumer)
}
// ParseIndexFromSite downloads sitemap index from a site, parses it and for each sitemap
// index entry calls the consumer's function.
func ParseIndexFromSite(sitemapURL string, consumer IndexEntryConsumer) error {
res, err := http.Get(sitemapURL)
if err != nil {
return err
}
defer res.Body.Close()
return ParseIndex(res.Body, consumer)
}
gopher-parse-sitemap-0.1/sitemap_impl.go 0000664 0000000 0000000 00000002240 13545335254 0020427 0 ustar 00root root 0000000 0000000 package sitemap
import (
"encoding/xml"
"io"
)
func entryParser(decoder *xml.Decoder, se *xml.StartElement, consume EntryConsumer) error {
if se.Name.Local == "url" {
entry := newSitemapEntry()
decodeError := decoder.DecodeElement(entry, se)
if decodeError != nil {
return decodeError
}
consume(entry)
}
return nil
}
func indexEntryParser(decoder *xml.Decoder, se *xml.StartElement, consume IndexEntryConsumer) error {
if se.Name.Local == "sitemap" {
entry := new(sitemapIndexEntry)
decodeError := decoder.DecodeElement(entry, se)
if decodeError != nil {
return decodeError
}
consume(entry)
}
return nil
}
type elementParser func(*xml.Decoder, *xml.StartElement) error
func parseLoop(reader io.Reader, parser elementParser) error {
decoder := xml.NewDecoder(reader)
for {
t, tokenError := decoder.Token()
if tokenError == io.EOF {
break
} else if tokenError != nil {
return tokenError
}
se, ok := t.(xml.StartElement)
if !ok {
continue
}
parserError := parser(decoder, &se)
if parserError != nil {
return parserError
}
}
return nil
}
gopher-parse-sitemap-0.1/sitemap_test.go 0000664 0000000 0000000 00000004710 13545335254 0020451 0 ustar 00root root 0000000 0000000 package sitemap
import (
"fmt"
"io/ioutil"
"strings"
"testing"
"time"
)
/*
* Examples
*/
func ExampleParseFromFile() {
err := ParseFromFile("./testdata/sitemap.xml", func(e Entry) error {
fmt.Println(e.GetLocation())
return nil
})
if err != nil {
panic(err)
}
}
func ExampleParseIndexFromFile() {
result := make([]string, 0, 0)
err := ParseIndexFromFile("./testdata/sitemap-index.xml", func(e IndexEntry) error {
result = append(result, e.GetLocation())
return nil
})
if err != nil {
panic(err)
}
}
/*
* Public API tests
*/
func TestParseSitemap(t *testing.T) {
var (
counter int
sb strings.Builder
)
err := ParseFromFile("./testdata/sitemap.xml", func(e Entry) error {
counter++
fmt.Fprintln(&sb, e.GetLocation())
lastmod := e.GetLastModified()
if lastmod != nil {
fmt.Fprintln(&sb, lastmod.Format(time.RFC3339))
}
fmt.Fprintln(&sb, e.GetChangeFrequency())
fmt.Fprintln(&sb, e.GetPriority())
return nil
})
if err != nil {
t.Errorf("Parsing failed with error %s", err)
}
if counter != 4 {
t.Errorf("Expected 4 elements, but given only %d", counter)
}
expected, err := ioutil.ReadFile("./testdata/sitemap.golden")
if err != nil {
t.Errorf("Can't read golden file due to %s", err)
}
if sb.String() != string(expected) {
t.Error("Unxepected result")
}
}
func TestParseSitemapIndex(t *testing.T) {
var (
counter int
sb strings.Builder
)
err := ParseIndexFromFile("./testdata/sitemap-index.xml", func(e IndexEntry) error {
counter++
fmt.Fprintln(&sb, e.GetLocation())
lastmod := e.GetLastModified()
if lastmod != nil {
fmt.Fprintln(&sb, lastmod.Format(time.RFC3339))
}
return nil
})
if err != nil {
t.Errorf("Parsing failed with error %s", err)
}
if counter != 3 {
t.Errorf("Expected 3 elements, but given only %d", counter)
}
expected, err := ioutil.ReadFile("./testdata/sitemap-index.golden")
if err != nil {
t.Errorf("Can't read golden file due to %s", err)
}
if sb.String() != string(expected) {
t.Error("Unxepected result")
}
}
/*
* Private API tests
*/
func TestParseShortDateTime(t *testing.T) {
res := parseDateTime("2015-05-07")
if res == nil {
t.Error("Date time was't parsed")
return
}
if res.Year() != 2015 || res.Month() != 05 || res.Day() != 07 {
t.Errorf("Date was parsed wrong %s", res.Format(time.RFC3339))
}
}
gopher-parse-sitemap-0.1/sitemap_types.go 0000664 0000000 0000000 00000002575 13545335254 0020645 0 ustar 00root root 0000000 0000000 package sitemap
import "time"
type sitemapEntry struct {
Location string `xml:"loc"`
LastModified string `xml:"lastmod,omitempy"`
ChangeFrequency Frequency `xml:"changefreq,omitempty"`
Priority float32 `xml:"priority,omitempty"`
}
func newSitemapEntry() *sitemapEntry {
return &sitemapEntry{ChangeFrequency: Always, Priority: 0.5}
}
func (e *sitemapEntry) GetLocation() string {
return e.Location
}
func (e *sitemapEntry) GetLastModified() *time.Time {
return parseDateTime(e.LastModified)
}
func (e *sitemapEntry) GetChangeFrequency() Frequency {
return e.ChangeFrequency
}
func (e *sitemapEntry) GetPriority() float32 {
return e.Priority
}
type sitemapIndexEntry struct {
Location string `xml:"loc"`
LastModified string `xml:"lastmod,omitempty"`
}
func newSitemapIndexEntry() *sitemapIndexEntry {
return &sitemapIndexEntry{}
}
func (e *sitemapIndexEntry) GetLocation() string {
return e.Location
}
func (e *sitemapIndexEntry) GetLastModified() *time.Time {
return parseDateTime(e.LastModified)
}
func parseDateTime(value string) *time.Time {
if value == "" {
return nil
}
t, err := time.Parse(time.RFC3339, value)
if err != nil {
// second chance
// try parse as short format
t, err = time.Parse("2006-01-02", value)
if err != nil {
return nil
}
}
return &t
}
gopher-parse-sitemap-0.1/testdata/ 0000775 0000000 0000000 00000000000 13545335254 0017230 5 ustar 00root root 0000000 0000000 gopher-parse-sitemap-0.1/testdata/sitemap-index.golden 0000664 0000000 0000000 00000000237 13545335254 0023173 0 ustar 00root root 0000000 0000000 http://www.example.com/sitemap1.xml.gz
2004-10-01T18:23:17Z
http://www.example.com/sitemap2.xml.gz
2005-01-01T00:00:00Z
http://www.example.com/sitemap3.xml.gz
gopher-parse-sitemap-0.1/testdata/sitemap-index.xml 0000664 0000000 0000000 00000000706 13545335254 0022524 0 ustar 00root root 0000000 0000000
http://www.example.com/sitemap1.xml.gz
2004-10-01T18:23:17+00:00
http://www.example.com/sitemap2.xml.gz
2005-01-01
http://www.example.com/sitemap3.xml.gz
gopher-parse-sitemap-0.1/testdata/sitemap.golden 0000664 0000000 0000000 00000000314 13545335254 0022062 0 ustar 00root root 0000000 0000000 http://HOST/
always
0.5
http://HOST/tools/
2015-05-07T19:13:09+09:00
always
0.5
http://HOST/contribution-to-oss/
2015-05-07T00:00:00Z
monthly
0.5
http://HOST/page-1/
2015-05-07T19:13:09+09:00
monthly
0.9
gopher-parse-sitemap-0.1/testdata/sitemap.xml 0000664 0000000 0000000 00000001101 13545335254 0021405 0 ustar 00root root 0000000 0000000
http://HOST/
http://HOST/tools/
2015-05-07T19:13:09+09:00
http://HOST/contribution-to-oss/
2015-05-07
monthly
http://HOST/page-1/
2015-05-07T19:13:09+09:00
monthly
0.9