⬆️ Update dependencies
This commit is contained in:
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
+15
-199
@@ -1,202 +1,18 @@
|
||||
Copyright (c) 2020 Josh Baker
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
# B-tree Path Hints
|
||||
|
||||
I use a thing I call path hints in my B-tree [C](https://github.com/tidwall/btree.c) and [Go](https://github.com/tidwall/btree) implementations. It's a search optimization.
|
||||
|
||||
## The B-tree
|
||||
|
||||
A standard [B-tree](https://en.wikipedia.org/wiki/B-tree) is an ordered tree-based data structure that stores its items in nodes. The B-tree has a single root node, which may have children nodes, and those children nodes may also have children nodes.
|
||||
|
||||
<img width="322" alt="image" src="https://user-images.githubusercontent.com/1156077/127664015-14ca38bb-1a3b-4d2f-80ff-27be0bd3d886.png">
|
||||
|
||||
Searching for items in a B-tree is fast. [O(log N)](https://en.wikipedia.org/wiki/Big_O_notation) to be exact.
|
||||
This is because the [binary search algorithm](https://en.wikipedia.org/wiki/Binary_search_algorithm) is used.
|
||||
|
||||
A binary search works by first comparing the item at the middle-most index of the root node with the target item.
|
||||
If the middle item is greater than the target item, then it divides the node in two and does the binary search on the left part of the node. If the middle is less, it searches the right part. And so on. If the target item is found, then the search stop. If the item is not found, then the search is passed to the child node at the appropriate index. This traversal terminates when item is found or there are no more child nodes.
|
||||
|
||||
<img width="600" alt="image" src="https://user-images.githubusercontent.com/1156077/127664822-6ab4f8f6-8ab5-477e-8e17-f52346f02819.png">
|
||||
|
||||
## The Path
|
||||
|
||||
Each index is a component of the path to the item (or where the item should be stored, if it does not exist in the tree).
|
||||
|
||||
Take the first example image. The item 9 is at path “1/0”. The item 16 is at path “1”. The item 21 is at path “2/1”. The item 5 is at path “0/2”.
|
||||
|
||||
## The Path Hint
|
||||
|
||||
A Path Hint is a predefined path that is provided to B-tree operations. It’s just a hint that says, “Hey B-tree, instead of starting your binary search with the middle index, start with what I provide you. My path may be wrong, and if so please provide me with the correct path so I get it right the next time.”
|
||||
|
||||
I’ve found using path hints can lead to a little performance increase of 150% - 300%. This is because in real-world cases the items that I’m working with are usually nearby each other in the tree.
|
||||
|
||||
Take for example inserting a group of timeseries points. They may often be received as chucks of near-contiguous items.
|
||||
Or, I'm sequentially inserting an ordered group of rows somewhere in the middle of a table.
|
||||
Or, I have a Redis-style key/value store, where the keys look have the common structure “user:98512:name”, “user:98512:email”, and I want to update a bunch of values for specified user.
|
||||
Using a path hint may help to avoid the unnecessary binary searching in each of these examples.
|
||||
|
||||
While I may see a 3x boost in when the path hint is right on, I'll only see around 5% decrease when the path hint is totally wrong.
|
||||
|
||||
## Using a Path Hint
|
||||
|
||||
All of the functions that take in a path hint argument mutate the path hint argument.
|
||||
|
||||
For single-threaded programs, it’s possible to use one shared path hint per B-tree for the life of the program.
|
||||
For multi-threaded programs, I find it best to use one path hint per B-tree , per thread.
|
||||
For server-client programs, one path hint per B-tree, per client should suffice.
|
||||
|
||||
+394
-50
@@ -1,23 +1,232 @@
|
||||
BTree implementation for Go
|
||||
===========================
|
||||
# btree
|
||||
|
||||
|
||||
[](https://godoc.org/github.com/tidwall/btree)
|
||||
|
||||
This package provides an in-memory B-Tree implementation for Go, useful as
|
||||
an ordered, mutable data structure.
|
||||
An efficient [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go.
|
||||
|
||||
This is a fork of the wonderful [google/btree](https://github.com/google/btree) package. It's has all the same great features and adds a few more.
|
||||
## Features
|
||||
|
||||
- Descend* functions for iterating backwards.
|
||||
- Iteration performance boost.
|
||||
- User defined context.
|
||||
- Support for [Generics](#generics) (Go 1.18+).
|
||||
- `Map` and `Set` types for ordered key-value maps and sets,
|
||||
- Fast bulk loading for pre-ordered data using the `Load()` method.
|
||||
- `Copy()` method with copy-on-write support.
|
||||
- Thread-safe operations.
|
||||
- [Path hinting](PATH_HINT.md) optimization for operations with nearby keys.
|
||||
|
||||
User defined context
|
||||
--------------------
|
||||
This is a great new feature that allows for entering the same item into multiple B-trees, and each B-tree have a different ordering formula.
|
||||
## Using
|
||||
|
||||
For example:
|
||||
To start using this package, install Go and run:
|
||||
|
||||
```sh
|
||||
$ go get github.com/tidwall/btree
|
||||
```
|
||||
|
||||
## B-tree types
|
||||
|
||||
This package includes the following types of B-trees:
|
||||
|
||||
- [`btree.Map`](#btreemap):
|
||||
A fast B-tree for storing ordered key value pairs.
|
||||
Go 1.18+
|
||||
- [`btree.Set`](#btreeset):
|
||||
Like `Map`, but only for storing keys.
|
||||
Go 1.18+
|
||||
- [`btree.BTreeG`](#btreegeneric):
|
||||
A feature-rich B-tree for storing data using a custom comparator.
|
||||
Go 1.18+
|
||||
- [`btree.BTree`](#btreebtree):
|
||||
Like `BTreeG` but uses the `interface{}` type for data. Backwards compatible.
|
||||
Go 1.16+
|
||||
|
||||
### btree.Map
|
||||
|
||||
```go
|
||||
// Basic
|
||||
Set(key, value) // insert or replace an item
|
||||
Get(key, value) // get an existing item
|
||||
Delete(key) // delete an item
|
||||
Len() // return the number of items in the map
|
||||
|
||||
// Iteration
|
||||
Scan(iter) // scan items in ascending order
|
||||
Reverse(iter) // scan items in descending order
|
||||
Ascend(key, iter) // scan items in ascending order that are >= to key
|
||||
Descend(key, iter) // scan items in descending order that are <= to key.
|
||||
Iter() // returns a read-only iterator for for-loops.
|
||||
|
||||
// Array-like operations
|
||||
GetAt(index) // returns the item at index
|
||||
DeleteAt(index) // deletes the item at index
|
||||
|
||||
// Bulk-loading
|
||||
Load(key, value) // load presorted items into tree
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/tidwall/btree"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create a map
|
||||
var users btree.Map[string, string]
|
||||
|
||||
// add some users
|
||||
users.Set("user:4", "Andrea")
|
||||
users.Set("user:6", "Andy")
|
||||
users.Set("user:2", "Andy")
|
||||
users.Set("user:1", "Jane")
|
||||
users.Set("user:5", "Janet")
|
||||
users.Set("user:3", "Steve")
|
||||
|
||||
// Iterate over the maps and print each user
|
||||
users.Scan(func(key, value string) bool {
|
||||
fmt.Printf("%s %s\n", key, value)
|
||||
return true
|
||||
})
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Delete a couple
|
||||
users.Delete("user:5")
|
||||
users.Delete("user:1")
|
||||
|
||||
// print the map again
|
||||
users.Scan(func(key, value string) bool {
|
||||
fmt.Printf("%s %s\n", key, value)
|
||||
return true
|
||||
})
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Output:
|
||||
// user:1 Jane
|
||||
// user:2 Andy
|
||||
// user:3 Steve
|
||||
// user:4 Andrea
|
||||
// user:5 Janet
|
||||
// user:6 Andy
|
||||
//
|
||||
// user:2 Andy
|
||||
// user:3 Steve
|
||||
// user:4 Andrea
|
||||
// user:6 Andy
|
||||
}
|
||||
```
|
||||
|
||||
### btree.Set
|
||||
|
||||
```go
|
||||
// Basic
|
||||
Insert(key) // insert an item
|
||||
Contains(key) // test if item exists
|
||||
Delete(key) // delete an item
|
||||
Len() // return the number of items in the set
|
||||
|
||||
// Iteration
|
||||
Scan(iter) // scan items in ascending order
|
||||
Reverse(iter) // scan items in descending order
|
||||
Ascend(key, iter) // scan items in ascending order that are >= to key
|
||||
Descend(key, iter) // scan items in descending order that are <= to key.
|
||||
Iter() // returns a read-only iterator for for-loops.
|
||||
|
||||
// Array-like operations
|
||||
GetAt(index) // returns the item at index
|
||||
DeleteAt(index) // deletes the item at index
|
||||
|
||||
// Bulk-loading
|
||||
Load(key) // load presorted item into tree
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/tidwall/btree"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create a set
|
||||
var names btree.Set[string]
|
||||
|
||||
// add some names
|
||||
names.Insert("Jane")
|
||||
names.Insert("Andrea")
|
||||
names.Insert("Steve")
|
||||
names.Insert("Andy")
|
||||
names.Insert("Janet")
|
||||
names.Insert("Andy")
|
||||
|
||||
// Iterate over the maps and print each user
|
||||
names.Scan(func(key string) bool {
|
||||
fmt.Printf("%s\n", key)
|
||||
return true
|
||||
})
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Delete a couple
|
||||
names.Delete("Steve")
|
||||
names.Delete("Andy")
|
||||
|
||||
// print the map again
|
||||
names.Scan(func(key string) bool {
|
||||
fmt.Printf("%s\n", key)
|
||||
return true
|
||||
})
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Output:
|
||||
// Andrea
|
||||
// Andy
|
||||
// Jane
|
||||
// Janet
|
||||
// Steve
|
||||
//
|
||||
// Andrea
|
||||
// Jane
|
||||
// Janet
|
||||
}
|
||||
```
|
||||
|
||||
### btree.BTreeG
|
||||
|
||||
```go
|
||||
// Basic
|
||||
Set(item) // insert or replace an item
|
||||
Get(item) // get an existing item
|
||||
Delete(item) // delete an item
|
||||
Len() // return the number of items in the btree
|
||||
|
||||
// Iteration
|
||||
Scan(iter) // scan items in ascending order
|
||||
Reverse(iter) // scan items in descending order
|
||||
Ascend(key, iter) // scan items in ascending order that are >= to key
|
||||
Descend(key, iter) // scan items in descending order that are <= to key.
|
||||
Iter() // returns a read-only iterator for for-loops.
|
||||
|
||||
// Array-like operations
|
||||
GetAt(index) // returns the item at index
|
||||
DeleteAt(index) // deletes the item at index
|
||||
|
||||
// Bulk-loading
|
||||
Load(item) // load presorted items into tree
|
||||
|
||||
// Path hinting
|
||||
SetHint(item, *hint) // insert or replace an existing item
|
||||
GetHint(item, *hint) // get an existing item
|
||||
DeleteHint(item, *hint) // delete an item
|
||||
|
||||
// Copy-on-write
|
||||
Copy() // copy the btree
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```go
|
||||
package main
|
||||
@@ -32,30 +241,154 @@ type Item struct {
|
||||
Key, Val string
|
||||
}
|
||||
|
||||
func (i1 *Item) Less(item btree.Item, ctx interface{}) bool {
|
||||
i2 := item.(*Item)
|
||||
switch tag := ctx.(type) {
|
||||
case string:
|
||||
if tag == "vals" {
|
||||
if i1.Val < i2.Val {
|
||||
return true
|
||||
} else if i1.Val > i2.Val {
|
||||
return false
|
||||
}
|
||||
// Both vals are equal so we should fall though
|
||||
// and let the key comparison take over.
|
||||
}
|
||||
// byKeys is a comparison function that compares item keys and returns true
|
||||
// when a is less than b.
|
||||
func byKeys(a, b Item) bool {
|
||||
return a.Key < b.Key
|
||||
}
|
||||
|
||||
// byVals is a comparison function that compares item values and returns true
|
||||
// when a is less than b.
|
||||
func byVals(a, b Item) bool {
|
||||
if a.Val < b.Val {
|
||||
return true
|
||||
}
|
||||
return i1.Key < i2.Key
|
||||
if a.Val > b.Val {
|
||||
return false
|
||||
}
|
||||
// Both vals are equal so we should fall though
|
||||
// and let the key comparison take over.
|
||||
return byKeys(a, b)
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
// Create a tree for keys and a tree for values.
|
||||
// The "keys" tree will be sorted on the Keys field.
|
||||
// The "values" tree will be sorted on the Values field.
|
||||
keys := btree.New(16, "keys")
|
||||
vals := btree.New(16, "vals")
|
||||
keys := btree.NewBTreeG[Item](byKeys)
|
||||
vals := btree.NewBTreeG[Item](byVals)
|
||||
|
||||
// Create some items.
|
||||
users := []Item{
|
||||
Item{Key: "user:1", Val: "Jane"},
|
||||
Item{Key: "user:2", Val: "Andy"},
|
||||
Item{Key: "user:3", Val: "Steve"},
|
||||
Item{Key: "user:4", Val: "Andrea"},
|
||||
Item{Key: "user:5", Val: "Janet"},
|
||||
Item{Key: "user:6", Val: "Andy"},
|
||||
}
|
||||
|
||||
// Insert each user into both trees
|
||||
for _, user := range users {
|
||||
keys.Set(user)
|
||||
vals.Set(user)
|
||||
}
|
||||
|
||||
// Iterate over each user in the key tree
|
||||
keys.Scan(func(item Item) bool {
|
||||
fmt.Printf("%s %s\n", item.Key, item.Val)
|
||||
return true
|
||||
})
|
||||
fmt.Printf("\n")
|
||||
|
||||
// Iterate over each user in the val tree
|
||||
vals.Scan(func(item Item) bool {
|
||||
fmt.Printf("%s %s\n", item.Key, item.Val)
|
||||
return true
|
||||
})
|
||||
|
||||
// Output:
|
||||
// user:1 Jane
|
||||
// user:2 Andy
|
||||
// user:3 Steve
|
||||
// user:4 Andrea
|
||||
// user:5 Janet
|
||||
// user:6 Andy
|
||||
//
|
||||
// user:4 Andrea
|
||||
// user:2 Andy
|
||||
// user:6 Andy
|
||||
// user:1 Jane
|
||||
// user:5 Janet
|
||||
// user:3 Steve
|
||||
}
|
||||
```
|
||||
|
||||
### btree.BTree
|
||||
|
||||
```go
|
||||
// Basic
|
||||
Set(item) // insert or replace an item
|
||||
Get(item) // get an existing item
|
||||
Delete(item) // delete an item
|
||||
Len() // return the number of items in the btree
|
||||
|
||||
// Iteration
|
||||
Scan(iter) // scan items in ascending order
|
||||
Reverse(iter) // scan items in descending order
|
||||
Ascend(key, iter) // scan items in ascending order that are >= to key
|
||||
Descend(key, iter) // scan items in descending order that are <= to key.
|
||||
Iter() // returns a read-only iterator for for-loops.
|
||||
|
||||
// Array-like operations
|
||||
GetAt(index) // returns the item at index
|
||||
DeleteAt(index) // deletes the item at index
|
||||
|
||||
// Bulk-loading
|
||||
Load(item) // load presorted items into tree
|
||||
|
||||
// Path hinting
|
||||
SetHint(item, *hint) // insert or replace an existing item
|
||||
GetHint(item, *hint) // get an existing item
|
||||
DeleteHint(item, *hint) // delete an item
|
||||
|
||||
// Copy-on-write
|
||||
Copy() // copy the btree
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/tidwall/btree"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
Key, Val string
|
||||
}
|
||||
|
||||
// byKeys is a comparison function that compares item keys and returns true
|
||||
// when a is less than b.
|
||||
func byKeys(a, b interface{}) bool {
|
||||
i1, i2 := a.(*Item), b.(*Item)
|
||||
return i1.Key < i2.Key
|
||||
}
|
||||
|
||||
// byVals is a comparison function that compares item values and returns true
|
||||
// when a is less than b.
|
||||
func byVals(a, b interface{}) bool {
|
||||
i1, i2 := a.(*Item), b.(*Item)
|
||||
if i1.Val < i2.Val {
|
||||
return true
|
||||
}
|
||||
if i1.Val > i2.Val {
|
||||
return false
|
||||
}
|
||||
// Both vals are equal so we should fall though
|
||||
// and let the key comparison take over.
|
||||
return byKeys(a, b)
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Create a tree for keys and a tree for values.
|
||||
// The "keys" tree will be sorted on the Keys field.
|
||||
// The "values" tree will be sorted on the Values field.
|
||||
keys := btree.New(byKeys)
|
||||
vals := btree.New(byVals)
|
||||
|
||||
// Create some items.
|
||||
users := []*Item{
|
||||
@@ -69,12 +402,12 @@ func main() {
|
||||
|
||||
// Insert each user into both trees
|
||||
for _, user := range users {
|
||||
keys.ReplaceOrInsert(user)
|
||||
vals.ReplaceOrInsert(user)
|
||||
keys.Set(user)
|
||||
vals.Set(user)
|
||||
}
|
||||
|
||||
// Iterate over each user in the key tree
|
||||
keys.Ascend(func(item btree.Item) bool {
|
||||
keys.Ascend(nil, func(item interface{}) bool {
|
||||
kvi := item.(*Item)
|
||||
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
|
||||
return true
|
||||
@@ -82,26 +415,37 @@ func main() {
|
||||
|
||||
fmt.Printf("\n")
|
||||
// Iterate over each user in the val tree
|
||||
vals.Ascend(func(item btree.Item) bool {
|
||||
vals.Ascend(nil, func(item interface{}) bool {
|
||||
kvi := item.(*Item)
|
||||
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
|
||||
return true
|
||||
})
|
||||
|
||||
// Output:
|
||||
// user:1 Jane
|
||||
// user:2 Andy
|
||||
// user:3 Steve
|
||||
// user:4 Andrea
|
||||
// user:5 Janet
|
||||
// user:6 Andy
|
||||
//
|
||||
// user:4 Andrea
|
||||
// user:2 Andy
|
||||
// user:6 Andy
|
||||
// user:1 Jane
|
||||
// user:5 Janet
|
||||
// user:3 Steve
|
||||
}
|
||||
|
||||
// Should see the results
|
||||
/*
|
||||
user:1 Jane
|
||||
user:2 Andy
|
||||
user:3 Steve
|
||||
user:4 Andrea
|
||||
user:5 Janet
|
||||
user:6 Andy
|
||||
|
||||
user:4 Andrea
|
||||
user:2 Andy
|
||||
user:6 Andy
|
||||
user:1 Jane
|
||||
user:3 Steve
|
||||
*/
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
See [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark) for benchmark numbers.
|
||||
|
||||
## Contact
|
||||
|
||||
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||
|
||||
## License
|
||||
|
||||
Source code is available under the MIT [License](/LICENSE).
|
||||
|
||||
+221
-926
File diff suppressed because it is too large
Load Diff
+1303
File diff suppressed because it is too large
Load Diff
+1056
File diff suppressed because it is too large
Load Diff
+168
@@ -0,0 +1,168 @@
|
||||
package btree
|
||||
|
||||
type Set[K ordered] struct {
|
||||
base Map[K, struct{}]
|
||||
}
|
||||
|
||||
// Copy
|
||||
func (tr *Set[K]) Copy() *Set[K] {
|
||||
tr2 := new(Set[K])
|
||||
tr2.base = *tr.base.Copy()
|
||||
return tr2
|
||||
}
|
||||
|
||||
// Insert an item
|
||||
func (tr *Set[K]) Insert(key K) {
|
||||
tr.base.Set(key, struct{}{})
|
||||
}
|
||||
|
||||
func (tr *Set[K]) Scan(iter func(key K) bool) {
|
||||
tr.base.Scan(func(key K, value struct{}) bool {
|
||||
return iter(key)
|
||||
})
|
||||
}
|
||||
|
||||
// Get a value for key
|
||||
func (tr *Set[K]) Contains(key K) bool {
|
||||
_, ok := tr.base.Get(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Len returns the number of items in the tree
|
||||
func (tr *Set[K]) Len() int {
|
||||
return tr.base.Len()
|
||||
}
|
||||
|
||||
// Delete an item
|
||||
func (tr *Set[K]) Delete(key K) {
|
||||
tr.base.Delete(key)
|
||||
}
|
||||
|
||||
// Ascend the tree within the range [pivot, last]
|
||||
// Pass nil for pivot to scan all item in ascending order
|
||||
// Return false to stop iterating
|
||||
func (tr *Set[K]) Ascend(pivot K, iter func(key K) bool) {
|
||||
tr.base.Ascend(pivot, func(key K, value struct{}) bool {
|
||||
return iter(key)
|
||||
})
|
||||
}
|
||||
|
||||
func (tr *Set[K]) Reverse(iter func(key K) bool) {
|
||||
tr.base.Reverse(func(key K, value struct{}) bool {
|
||||
return iter(key)
|
||||
})
|
||||
}
|
||||
|
||||
// Descend the tree within the range [pivot, first]
|
||||
// Pass nil for pivot to scan all item in descending order
|
||||
// Return false to stop iterating
|
||||
func (tr *Set[K]) Descend(pivot K, iter func(key K) bool) {
|
||||
tr.base.Descend(pivot, func(key K, value struct{}) bool {
|
||||
return iter(key)
|
||||
})
|
||||
}
|
||||
|
||||
// Load is for bulk loading pre-sorted items
|
||||
func (tr *Set[K]) Load(key K) {
|
||||
tr.base.Load(key, struct{}{})
|
||||
}
|
||||
|
||||
// Min returns the minimum item in tree.
|
||||
// Returns nil if the treex has no items.
|
||||
func (tr *Set[K]) Min() (K, bool) {
|
||||
key, _, ok := tr.base.Min()
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// Max returns the maximum item in tree.
|
||||
// Returns nil if the tree has no items.
|
||||
func (tr *Set[K]) Max() (K, bool) {
|
||||
key, _, ok := tr.base.Max()
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// PopMin removes the minimum item in tree and returns it.
|
||||
// Returns nil if the tree has no items.
|
||||
func (tr *Set[K]) PopMin() (K, bool) {
|
||||
key, _, ok := tr.base.PopMin()
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// PopMax removes the maximum item in tree and returns it.
|
||||
// Returns nil if the tree has no items.
|
||||
func (tr *Set[K]) PopMax() (K, bool) {
|
||||
key, _, ok := tr.base.PopMax()
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// GetAt returns the value at index.
|
||||
// Return nil if the tree is empty or the index is out of bounds.
|
||||
func (tr *Set[K]) GetAt(index int) (K, bool) {
|
||||
key, _, ok := tr.base.GetAt(index)
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// DeleteAt deletes the item at index.
|
||||
// Return nil if the tree is empty or the index is out of bounds.
|
||||
func (tr *Set[K]) DeleteAt(index int) (K, bool) {
|
||||
key, _, ok := tr.base.DeleteAt(index)
|
||||
return key, ok
|
||||
}
|
||||
|
||||
// Height returns the height of the tree.
|
||||
// Returns zero if tree has no items.
|
||||
func (tr *Set[K]) Height() int {
|
||||
return tr.base.Height()
|
||||
}
|
||||
|
||||
// SetIter represents an iterator for btree.Set
|
||||
type SetIter[K ordered] struct {
|
||||
base MapIter[K, struct{}]
|
||||
}
|
||||
|
||||
// Iter returns a read-only iterator.
|
||||
func (tr *Set[K]) Iter() SetIter[K] {
|
||||
return SetIter[K]{tr.base.Iter()}
|
||||
}
|
||||
|
||||
// Seek to item greater-or-equal-to key.
|
||||
// Returns false if there was no item found.
|
||||
func (iter *SetIter[K]) Seek(key K) bool {
|
||||
return iter.base.Seek(key)
|
||||
}
|
||||
|
||||
// First moves iterator to first item in tree.
|
||||
// Returns false if the tree is empty.
|
||||
func (iter *SetIter[K]) First() bool {
|
||||
return iter.base.First()
|
||||
}
|
||||
|
||||
// Last moves iterator to last item in tree.
|
||||
// Returns false if the tree is empty.
|
||||
func (iter *SetIter[K]) Last() bool {
|
||||
return iter.base.Last()
|
||||
}
|
||||
|
||||
// Next moves iterator to the next item in iterator.
|
||||
// Returns false if the tree is empty or the iterator is at the end of
|
||||
// the tree.
|
||||
func (iter *SetIter[K]) Next() bool {
|
||||
return iter.base.Next()
|
||||
}
|
||||
|
||||
// Prev moves iterator to the previous item in iterator.
|
||||
// Returns false if the tree is empty or the iterator is at the beginning of
|
||||
// the tree.
|
||||
func (iter *SetIter[K]) Prev() bool {
|
||||
return iter.base.Prev()
|
||||
}
|
||||
|
||||
// Key returns the current iterator item key.
|
||||
func (iter *SetIter[K]) Key() K {
|
||||
return iter.base.Key()
|
||||
}
|
||||
|
||||
// Keys returns all the keys in order.
|
||||
func (tr *Set[K]) Keys() []K {
|
||||
return tr.base.Keys()
|
||||
}
|
||||
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
+7
-8
@@ -3,10 +3,8 @@
|
||||
src="logo.png"
|
||||
width="307" height="150" border="0" alt="BuntDB">
|
||||
<br>
|
||||
<a href="https://travis-ci.org/tidwall/buntdb"><img src="https://img.shields.io/travis/tidwall/buntdb.svg?style=flat-square" alt="Build Status"></a>
|
||||
<a href="http://gocover.io/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/coverage-95%25-brightgreen.svg?style=flat-square" alt="Code Coverage"></a>
|
||||
<a href="https://goreportcard.com/report/github.com/tidwall/buntdb"><img src="https://goreportcard.com/badge/github.com/tidwall/buntdb?style=flat-square" alt="Go Report Card"></a>
|
||||
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square" alt="Godoc"></a>
|
||||
<a href="https://github.com/tidwall/buntdb/blob/master/LICENSE"><img src="https://img.shields.io/github/license/tidwall/buntdb.svg?style=flat-square" alt="LICENSE"></a>
|
||||
</p>
|
||||
|
||||
BuntDB is a low-level, in-memory, key/value store in pure Go.
|
||||
@@ -29,7 +27,6 @@ Features
|
||||
- Flexible [iteration](#iterating) of data; ascending, descending, and ranges
|
||||
- [Durable append-only file](#append-only-file) format for persistence
|
||||
- Option to evict old items with an [expiration](#data-expiration) TTL
|
||||
- Tight codebase, under 2K loc using the `cloc` command
|
||||
- ACID semantics with locking [transactions](#transactions) that support rollbacks
|
||||
|
||||
|
||||
@@ -140,6 +137,7 @@ All keys/value pairs are ordered in the database by the key. To iterate over the
|
||||
err := db.View(func(tx *buntdb.Tx) error {
|
||||
err := tx.Ascend("", func(key, value string) bool {
|
||||
fmt.Printf("key: %s, value: %s\n", key, value)
|
||||
return true // continue iteration
|
||||
})
|
||||
return err
|
||||
})
|
||||
@@ -458,8 +456,9 @@ Any index can be put in descending order by wrapping it's less function with `bu
|
||||
|
||||
```go
|
||||
db.CreateIndex("last_name_age", "*",
|
||||
buntdb.IndexJSON("name.last"),
|
||||
buntdb.Desc(buntdb.IndexJSON("age")))
|
||||
buntdb.IndexJSON("name.last"),
|
||||
buntdb.Desc(buntdb.IndexJSON("age")),
|
||||
)
|
||||
```
|
||||
|
||||
This will create a multi value index where the last name is ascending and the age is descending.
|
||||
@@ -576,7 +575,7 @@ var config buntdb.Config
|
||||
if err := db.ReadConfig(&config); err != nil{
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := db.WriteConfig(config); err != nil{
|
||||
if err := db.SetConfig(config); err != nil{
|
||||
log.Fatal(err)
|
||||
}
|
||||
```
|
||||
|
||||
+338
-111
@@ -1,14 +1,16 @@
|
||||
// Package buntdb implements a low-level in-memory key/value store in pure Go.
|
||||
// It persists to disk, is ACID compliant, and uses locking for multiple
|
||||
// readers and a single writer. Bunt is ideal for projects that need
|
||||
// a dependable database, and favor speed over data size.
|
||||
// readers and a single writer. Bunt is ideal for projects that need a
|
||||
// dependable database, and favor speed over data size.
|
||||
package buntdb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -19,7 +21,7 @@ import (
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/grect"
|
||||
"github.com/tidwall/match"
|
||||
"github.com/tidwall/rtree"
|
||||
"github.com/tidwall/rtred"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -60,6 +62,8 @@ var (
|
||||
ErrTxIterating = errors.New("tx is iterating")
|
||||
)
|
||||
|
||||
const useAbsEx = true
|
||||
|
||||
// DB represents a collection of key-value pairs that persist on disk.
|
||||
// Transactions are used for all forms of data access to the DB.
|
||||
type DB struct {
|
||||
@@ -69,7 +73,7 @@ type DB struct {
|
||||
keys *btree.BTree // a tree of all item ordered by key
|
||||
exps *btree.BTree // a tree of items ordered by expiration
|
||||
idxs map[string]*index // the index trees.
|
||||
exmgr bool // indicates that expires manager is running.
|
||||
insIdxs []*index // a reuse buffer for gathering indexes
|
||||
flushes int // a count of the number of disk flushes
|
||||
closed bool // set when the database has been closed
|
||||
config Config // the database configuration
|
||||
@@ -122,10 +126,11 @@ type Config struct {
|
||||
// has been expired.
|
||||
OnExpired func(keys []string)
|
||||
|
||||
// OnExpiredSync will be called inside the same transaction that is performing
|
||||
// the deletion of expired items. If OnExpired is present then this callback
|
||||
// will not be called. If this callback is present, then the deletion of the
|
||||
// timeed-out item is the explicit responsibility of this callback.
|
||||
// OnExpiredSync will be called inside the same transaction that is
|
||||
// performing the deletion of expired items. If OnExpired is present then
|
||||
// this callback will not be called. If this callback is present, then the
|
||||
// deletion of the timeed-out item is the explicit responsibility of this
|
||||
// callback.
|
||||
OnExpiredSync func(key, value string, tx *Tx) error
|
||||
}
|
||||
|
||||
@@ -134,16 +139,13 @@ type exctx struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// Default number of btree degrees
|
||||
const btreeDegrees = 64
|
||||
|
||||
// Open opens a database at the provided path.
|
||||
// If the file does not exist then it will be created automatically.
|
||||
func Open(path string) (*DB, error) {
|
||||
db := &DB{}
|
||||
// initialize trees and indexes
|
||||
db.keys = btree.New(btreeDegrees, nil)
|
||||
db.exps = btree.New(btreeDegrees, &exctx{db})
|
||||
db.keys = btreeNew(lessCtx(nil))
|
||||
db.exps = btreeNew(lessCtx(&exctx{db}))
|
||||
db.idxs = make(map[string]*index)
|
||||
// initialize default configuration
|
||||
db.config = Config{
|
||||
@@ -203,10 +205,11 @@ func (db *DB) Save(wr io.Writer) error {
|
||||
defer db.mu.RUnlock()
|
||||
// use a buffered writer and flush every 4MB
|
||||
var buf []byte
|
||||
now := time.Now()
|
||||
// iterated through every item in the database and write to the buffer
|
||||
db.keys.Ascend(func(item btree.Item) bool {
|
||||
btreeAscend(db.keys, func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
buf = dbi.writeSetTo(buf)
|
||||
buf = dbi.writeSetTo(buf, now)
|
||||
if len(buf) > 1024*1024*4 {
|
||||
// flush when buffer is over 4MB
|
||||
_, err = wr.Write(buf)
|
||||
@@ -240,14 +243,15 @@ func (db *DB) Load(rd io.Reader) error {
|
||||
// cannot load into databases that persist to disk
|
||||
return ErrPersistenceActive
|
||||
}
|
||||
return db.readLoad(rd, time.Now())
|
||||
_, err := db.readLoad(rd, time.Now())
|
||||
return err
|
||||
}
|
||||
|
||||
// index represents a b-tree or r-tree index and also acts as the
|
||||
// b-tree/r-tree context for itself.
|
||||
type index struct {
|
||||
btr *btree.BTree // contains the items
|
||||
rtr *rtree.RTree // contains the items
|
||||
rtr *rtred.RTree // contains the items
|
||||
name string // name of the index
|
||||
pattern string // a required key pattern
|
||||
less func(a, b string) bool // less comparison function
|
||||
@@ -285,10 +289,10 @@ func (idx *index) clearCopy() *index {
|
||||
}
|
||||
// initialize with empty trees
|
||||
if nidx.less != nil {
|
||||
nidx.btr = btree.New(btreeDegrees, nidx)
|
||||
nidx.btr = btreeNew(lessCtx(nidx))
|
||||
}
|
||||
if nidx.rect != nil {
|
||||
nidx.rtr = rtree.New(nidx)
|
||||
nidx.rtr = rtred.New(nidx)
|
||||
}
|
||||
return nidx
|
||||
}
|
||||
@@ -297,20 +301,20 @@ func (idx *index) clearCopy() *index {
|
||||
func (idx *index) rebuild() {
|
||||
// initialize trees
|
||||
if idx.less != nil {
|
||||
idx.btr = btree.New(btreeDegrees, idx)
|
||||
idx.btr = btreeNew(lessCtx(idx))
|
||||
}
|
||||
if idx.rect != nil {
|
||||
idx.rtr = rtree.New(idx)
|
||||
idx.rtr = rtred.New(idx)
|
||||
}
|
||||
// iterate through all keys and fill the index
|
||||
idx.db.keys.Ascend(func(item btree.Item) bool {
|
||||
btreeAscend(idx.db.keys, func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
if !idx.match(dbi.key) {
|
||||
// does not match the pattern, conintue
|
||||
// does not match the pattern, continue
|
||||
return true
|
||||
}
|
||||
if idx.less != nil {
|
||||
idx.btr.ReplaceOrInsert(dbi)
|
||||
idx.btr.Set(dbi)
|
||||
}
|
||||
if idx.rect != nil {
|
||||
idx.rtr.Insert(dbi)
|
||||
@@ -456,16 +460,23 @@ func (db *DB) SetConfig(config Config) error {
|
||||
// will be replaced with the new one, and return the previous item.
|
||||
func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
|
||||
var pdbi *dbItem
|
||||
prev := db.keys.ReplaceOrInsert(item)
|
||||
// Generate a list of indexes that this item will be inserted in to.
|
||||
idxs := db.insIdxs
|
||||
for _, idx := range db.idxs {
|
||||
if idx.match(item.key) {
|
||||
idxs = append(idxs, idx)
|
||||
}
|
||||
}
|
||||
prev := db.keys.Set(item)
|
||||
if prev != nil {
|
||||
// A previous item was removed from the keys tree. Let's
|
||||
// fully delete this item from all indexes.
|
||||
pdbi = prev.(*dbItem)
|
||||
if pdbi.opts != nil && pdbi.opts.ex {
|
||||
// Remove it from the exipres tree.
|
||||
// Remove it from the expires tree.
|
||||
db.exps.Delete(pdbi)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
for _, idx := range idxs {
|
||||
if idx.btr != nil {
|
||||
// Remove it from the btree index.
|
||||
idx.btr.Delete(pdbi)
|
||||
@@ -479,21 +490,22 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
|
||||
if item.opts != nil && item.opts.ex {
|
||||
// The new item has eviction options. Add it to the
|
||||
// expires tree
|
||||
db.exps.ReplaceOrInsert(item)
|
||||
db.exps.Set(item)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
if !idx.match(item.key) {
|
||||
continue
|
||||
}
|
||||
for i, idx := range idxs {
|
||||
if idx.btr != nil {
|
||||
// Add new item to btree index.
|
||||
idx.btr.ReplaceOrInsert(item)
|
||||
idx.btr.Set(item)
|
||||
}
|
||||
if idx.rtr != nil {
|
||||
// Add new item to rtree index.
|
||||
idx.rtr.Insert(item)
|
||||
}
|
||||
// clear the index
|
||||
idxs[i] = nil
|
||||
}
|
||||
// reuse the index list slice
|
||||
db.insIdxs = idxs[:0]
|
||||
// we must return the previous item to the caller.
|
||||
return pdbi
|
||||
}
|
||||
@@ -514,6 +526,9 @@ func (db *DB) deleteFromDatabase(item *dbItem) *dbItem {
|
||||
db.exps.Delete(pdbi)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
if !idx.match(pdbi.key) {
|
||||
continue
|
||||
}
|
||||
if idx.btr != nil {
|
||||
// Remove it from the btree index.
|
||||
idx.btr.Delete(pdbi)
|
||||
@@ -557,9 +572,9 @@ func (db *DB) backgroundManager() {
|
||||
}
|
||||
}
|
||||
// produce a list of expired items that need removing
|
||||
db.exps.AscendLessThan(&dbItem{
|
||||
btreeAscendLessThan(db.exps, &dbItem{
|
||||
opts: &dbItemOpts{ex: true, exat: time.Now()},
|
||||
}, func(item btree.Item) bool {
|
||||
}, func(item interface{}) bool {
|
||||
expired = append(expired, item.(*dbItem))
|
||||
return true
|
||||
})
|
||||
@@ -674,8 +689,9 @@ func (db *DB) Shrink() error {
|
||||
}
|
||||
done = true
|
||||
var n int
|
||||
db.keys.AscendGreaterOrEqual(&dbItem{key: pivot},
|
||||
func(item btree.Item) bool {
|
||||
now := time.Now()
|
||||
btreeAscendGreaterOrEqual(db.keys, &dbItem{key: pivot},
|
||||
func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
// 1000 items or 64MB buffer
|
||||
if n > 1000 || len(buf) > 64*1024*1024 {
|
||||
@@ -683,7 +699,7 @@ func (db *DB) Shrink() error {
|
||||
done = false
|
||||
return false
|
||||
}
|
||||
buf = dbi.writeSetTo(buf)
|
||||
buf = dbi.writeSetTo(buf, now)
|
||||
n++
|
||||
return true
|
||||
},
|
||||
@@ -737,13 +753,13 @@ func (db *DB) Shrink() error {
|
||||
if err := db.file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Any failures below here is really bad. So just panic.
|
||||
if err := os.Rename(tmpname, fname); err != nil {
|
||||
panic(err)
|
||||
// Any failures below here are really bad. So just panic.
|
||||
if err := renameFile(tmpname, fname); err != nil {
|
||||
panicErr(err)
|
||||
}
|
||||
db.file, err = os.OpenFile(fname, os.O_CREATE|os.O_RDWR, 0666)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
panicErr(err)
|
||||
}
|
||||
pos, err := db.file.Seek(0, 2)
|
||||
if err != nil {
|
||||
@@ -754,46 +770,81 @@ func (db *DB) Shrink() error {
|
||||
}()
|
||||
}
|
||||
|
||||
var errValidEOF = errors.New("valid eof")
|
||||
func panicErr(err error) error {
|
||||
panic(fmt.Errorf("buntdb: %w", err))
|
||||
}
|
||||
|
||||
func renameFile(src, dest string) error {
|
||||
var err error
|
||||
if err = os.Rename(src, dest); err != nil {
|
||||
if runtime.GOOS == "windows" {
|
||||
if err = os.Remove(dest); err == nil {
|
||||
err = os.Rename(src, dest)
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// readLoad reads from the reader and loads commands into the database.
|
||||
// modTime is the modified time of the reader, should be no greater than
|
||||
// the current time.Now().
|
||||
func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
|
||||
// Returns the number of bytes of the last command read and the error if any.
|
||||
func (db *DB) readLoad(rd io.Reader, modTime time.Time) (n int64, err error) {
|
||||
defer func() {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
}()
|
||||
totalSize := int64(0)
|
||||
data := make([]byte, 4096)
|
||||
parts := make([]string, 0, 8)
|
||||
r := bufio.NewReader(rd)
|
||||
for {
|
||||
// peek at the first byte. If it's a 'nul' control character then
|
||||
// ignore it and move to the next byte.
|
||||
c, err := r.ReadByte()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
return totalSize, err
|
||||
}
|
||||
if c == 0 {
|
||||
// ignore nul control characters
|
||||
n += 1
|
||||
continue
|
||||
}
|
||||
if err := r.UnreadByte(); err != nil {
|
||||
return totalSize, err
|
||||
}
|
||||
|
||||
// read a single command.
|
||||
// first we should read the number of parts that the of the command
|
||||
cmdByteSize := int64(0)
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
if len(line) > 0 {
|
||||
// got an eof but also data. this should be an unexpected eof.
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
return totalSize, err
|
||||
}
|
||||
if line[0] != '*' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
cmdByteSize += int64(len(line))
|
||||
|
||||
// convert the string number to and int
|
||||
var n int
|
||||
if len(line) == 4 && line[len(line)-2] == '\r' {
|
||||
if line[1] < '0' || line[1] > '9' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
n = int(line[1] - '0')
|
||||
} else {
|
||||
if len(line) < 5 || line[len(line)-2] != '\r' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
for i := 1; i < len(line)-2; i++ {
|
||||
if line[i] < '0' || line[i] > '9' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
n = n*10 + int(line[i]-'0')
|
||||
}
|
||||
@@ -804,25 +855,26 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
|
||||
// read the number of bytes of the part.
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return err
|
||||
return totalSize, err
|
||||
}
|
||||
if line[0] != '$' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
cmdByteSize += int64(len(line))
|
||||
// convert the string number to and int
|
||||
var n int
|
||||
if len(line) == 4 && line[len(line)-2] == '\r' {
|
||||
if line[1] < '0' || line[1] > '9' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
n = int(line[1] - '0')
|
||||
} else {
|
||||
if len(line) < 5 || line[len(line)-2] != '\r' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
for i := 1; i < len(line)-2; i++ {
|
||||
if line[i] < '0' || line[i] > '9' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
n = n*10 + int(line[i]-'0')
|
||||
}
|
||||
@@ -836,67 +888,79 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
|
||||
data = make([]byte, dataln)
|
||||
}
|
||||
if _, err = io.ReadFull(r, data[:n+2]); err != nil {
|
||||
return err
|
||||
return totalSize, err
|
||||
}
|
||||
if data[n] != '\r' || data[n+1] != '\n' {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
// copy string
|
||||
parts = append(parts, string(data[:n]))
|
||||
cmdByteSize += int64(n + 2)
|
||||
}
|
||||
// finished reading the command
|
||||
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
if (parts[0][0] == 's' || parts[0][1] == 'S') &&
|
||||
if (parts[0][0] == 's' || parts[0][0] == 'S') &&
|
||||
(parts[0][1] == 'e' || parts[0][1] == 'E') &&
|
||||
(parts[0][2] == 't' || parts[0][2] == 'T') {
|
||||
// SET
|
||||
if len(parts) < 3 || len(parts) == 4 || len(parts) > 5 {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
if len(parts) == 5 {
|
||||
if strings.ToLower(parts[3]) != "ex" {
|
||||
return ErrInvalid
|
||||
arg := strings.ToLower(parts[3])
|
||||
if arg != "ex" && arg != "ae" {
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
ex, err := strconv.ParseInt(parts[4], 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
return totalSize, err
|
||||
}
|
||||
var exat time.Time
|
||||
now := time.Now()
|
||||
dur := (time.Duration(ex) * time.Second) - now.Sub(modTime)
|
||||
if dur > 0 {
|
||||
if arg == "ex" {
|
||||
dur := (time.Duration(ex) * time.Second) - now.Sub(modTime)
|
||||
exat = now.Add(dur)
|
||||
} else {
|
||||
exat = time.Unix(ex, 0)
|
||||
}
|
||||
if exat.After(now) {
|
||||
db.insertIntoDatabase(&dbItem{
|
||||
key: parts[1],
|
||||
val: parts[2],
|
||||
opts: &dbItemOpts{
|
||||
ex: true,
|
||||
exat: now.Add(dur),
|
||||
exat: exat,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
db.deleteFromDatabase(&dbItem{
|
||||
key: parts[1],
|
||||
})
|
||||
}
|
||||
} else {
|
||||
db.insertIntoDatabase(&dbItem{key: parts[1], val: parts[2]})
|
||||
}
|
||||
} else if (parts[0][0] == 'd' || parts[0][1] == 'D') &&
|
||||
} else if (parts[0][0] == 'd' || parts[0][0] == 'D') &&
|
||||
(parts[0][1] == 'e' || parts[0][1] == 'E') &&
|
||||
(parts[0][2] == 'l' || parts[0][2] == 'L') {
|
||||
// DEL
|
||||
if len(parts) != 2 {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
db.deleteFromDatabase(&dbItem{key: parts[1]})
|
||||
} else if (parts[0][0] == 'f' || parts[0][1] == 'F') &&
|
||||
} else if (parts[0][0] == 'f' || parts[0][0] == 'F') &&
|
||||
strings.ToLower(parts[0]) == "flushdb" {
|
||||
db.keys = btree.New(btreeDegrees, nil)
|
||||
db.exps = btree.New(btreeDegrees, &exctx{db})
|
||||
db.keys = btreeNew(lessCtx(nil))
|
||||
db.exps = btreeNew(lessCtx(&exctx{db}))
|
||||
db.idxs = make(map[string]*index)
|
||||
} else {
|
||||
return ErrInvalid
|
||||
return totalSize, ErrInvalid
|
||||
}
|
||||
totalSize += cmdByteSize
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// load reads entries from the append only database file and fills the database.
|
||||
@@ -909,14 +973,29 @@ func (db *DB) load() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.readLoad(db.file, fi.ModTime()); err != nil {
|
||||
return err
|
||||
}
|
||||
pos, err := db.file.Seek(0, 2)
|
||||
n, err := db.readLoad(db.file, fi.ModTime())
|
||||
if err != nil {
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
// The db file has ended mid-command, which is allowed but the
|
||||
// data file should be truncated to the end of the last valid
|
||||
// command
|
||||
if err := db.file.Truncate(n); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := db.file.Seek(n, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
db.lastaofsz = int(pos)
|
||||
var estaofsz int
|
||||
db.keys.Walk(func(items []interface{}) {
|
||||
for _, v := range items {
|
||||
estaofsz += v.(*dbItem).estAOFSetSize()
|
||||
}
|
||||
})
|
||||
db.lastaofsz += estaofsz
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1025,8 +1104,8 @@ func (tx *Tx) DeleteAll() error {
|
||||
}
|
||||
|
||||
// now reset the live database trees
|
||||
tx.db.keys = btree.New(btreeDegrees, nil)
|
||||
tx.db.exps = btree.New(btreeDegrees, &exctx{tx.db})
|
||||
tx.db.keys = btreeNew(lessCtx(nil))
|
||||
tx.db.exps = btreeNew(lessCtx(&exctx{tx.db}))
|
||||
tx.db.idxs = make(map[string]*index)
|
||||
|
||||
// finally re-create the indexes
|
||||
@@ -1136,18 +1215,37 @@ func (tx *Tx) Commit() error {
|
||||
if tx.wc.rbkeys != nil {
|
||||
tx.db.buf = append(tx.db.buf, "*1\r\n$7\r\nflushdb\r\n"...)
|
||||
}
|
||||
now := time.Now()
|
||||
// Each committed record is written to disk
|
||||
for key, item := range tx.wc.commitItems {
|
||||
if item == nil {
|
||||
tx.db.buf = (&dbItem{key: key}).writeDeleteTo(tx.db.buf)
|
||||
} else {
|
||||
tx.db.buf = item.writeSetTo(tx.db.buf)
|
||||
tx.db.buf = item.writeSetTo(tx.db.buf, now)
|
||||
}
|
||||
}
|
||||
// Flushing the buffer only once per transaction.
|
||||
// If this operation fails then the write did failed and we must
|
||||
// rollback.
|
||||
if _, err = tx.db.file.Write(tx.db.buf); err != nil {
|
||||
var n int
|
||||
n, err = tx.db.file.Write(tx.db.buf)
|
||||
if err != nil {
|
||||
if n > 0 {
|
||||
// There was a partial write to disk.
|
||||
// We are possibly out of disk space.
|
||||
// Delete the partially written bytes from the data file by
|
||||
// seeking to the previously known position and performing
|
||||
// a truncate operation.
|
||||
// At this point a syscall failure is fatal and the process
|
||||
// should be killed to avoid corrupting the file.
|
||||
pos, err := tx.db.file.Seek(-int64(n), 1)
|
||||
if err != nil {
|
||||
panicErr(err)
|
||||
}
|
||||
if err := tx.db.file.Truncate(pos); err != nil {
|
||||
panicErr(err)
|
||||
}
|
||||
}
|
||||
tx.rollbackInner()
|
||||
}
|
||||
if tx.db.config.SyncPolicy == Always {
|
||||
@@ -1196,16 +1294,59 @@ type dbItem struct {
|
||||
keyless bool // keyless item for scanning
|
||||
}
|
||||
|
||||
// estIntSize returns the string representions size.
|
||||
// Has the same result as len(strconv.Itoa(x)).
|
||||
func estIntSize(x int) int {
|
||||
n := 1
|
||||
if x < 0 {
|
||||
n++
|
||||
x *= -1
|
||||
}
|
||||
for x >= 10 {
|
||||
n++
|
||||
x /= 10
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func estArraySize(count int) int {
|
||||
return 1 + estIntSize(count) + 2
|
||||
}
|
||||
|
||||
func estBulkStringSize(s string) int {
|
||||
return 1 + estIntSize(len(s)) + 2 + len(s) + 2
|
||||
}
|
||||
|
||||
// estAOFSetSize returns an estimated number of bytes that this item will use
|
||||
// when stored in the aof file.
|
||||
func (dbi *dbItem) estAOFSetSize() int {
|
||||
var n int
|
||||
if dbi.opts != nil && dbi.opts.ex {
|
||||
n += estArraySize(5)
|
||||
n += estBulkStringSize("set")
|
||||
n += estBulkStringSize(dbi.key)
|
||||
n += estBulkStringSize(dbi.val)
|
||||
n += estBulkStringSize("ex")
|
||||
n += estBulkStringSize("99") // estimate two byte bulk string
|
||||
} else {
|
||||
n += estArraySize(3)
|
||||
n += estBulkStringSize("set")
|
||||
n += estBulkStringSize(dbi.key)
|
||||
n += estBulkStringSize(dbi.val)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func appendArray(buf []byte, count int) []byte {
|
||||
buf = append(buf, '*')
|
||||
buf = append(buf, strconv.FormatInt(int64(count), 10)...)
|
||||
buf = strconv.AppendInt(buf, int64(count), 10)
|
||||
buf = append(buf, '\r', '\n')
|
||||
return buf
|
||||
}
|
||||
|
||||
func appendBulkString(buf []byte, s string) []byte {
|
||||
buf = append(buf, '$')
|
||||
buf = append(buf, strconv.FormatInt(int64(len(s)), 10)...)
|
||||
buf = strconv.AppendInt(buf, int64(len(s)), 10)
|
||||
buf = append(buf, '\r', '\n')
|
||||
buf = append(buf, s...)
|
||||
buf = append(buf, '\r', '\n')
|
||||
@@ -1213,15 +1354,21 @@ func appendBulkString(buf []byte, s string) []byte {
|
||||
}
|
||||
|
||||
// writeSetTo writes an item as a single SET record to the a bufio Writer.
|
||||
func (dbi *dbItem) writeSetTo(buf []byte) []byte {
|
||||
func (dbi *dbItem) writeSetTo(buf []byte, now time.Time) []byte {
|
||||
if dbi.opts != nil && dbi.opts.ex {
|
||||
ex := dbi.opts.exat.Sub(time.Now()) / time.Second
|
||||
buf = appendArray(buf, 5)
|
||||
buf = appendBulkString(buf, "set")
|
||||
buf = appendBulkString(buf, dbi.key)
|
||||
buf = appendBulkString(buf, dbi.val)
|
||||
buf = appendBulkString(buf, "ex")
|
||||
buf = appendBulkString(buf, strconv.FormatUint(uint64(ex), 10))
|
||||
if useAbsEx {
|
||||
ex := dbi.opts.exat.Unix()
|
||||
buf = appendBulkString(buf, "ae")
|
||||
buf = appendBulkString(buf, strconv.FormatUint(uint64(ex), 10))
|
||||
} else {
|
||||
ex := dbi.opts.exat.Sub(now) / time.Second
|
||||
buf = appendBulkString(buf, "ex")
|
||||
buf = appendBulkString(buf, strconv.FormatUint(uint64(ex), 10))
|
||||
}
|
||||
} else {
|
||||
buf = appendArray(buf, 3)
|
||||
buf = appendBulkString(buf, "set")
|
||||
@@ -1264,8 +1411,7 @@ func (dbi *dbItem) expiresAt() time.Time {
|
||||
// to note that the ctx parameter is used to help with determine which
|
||||
// formula to use on an item. Each b-tree should use a different ctx when
|
||||
// sharing the same item.
|
||||
func (dbi *dbItem) Less(item btree.Item, ctx interface{}) bool {
|
||||
dbi2 := item.(*dbItem)
|
||||
func (dbi *dbItem) Less(dbi2 *dbItem, ctx interface{}) bool {
|
||||
switch ctx := ctx.(type) {
|
||||
case *exctx:
|
||||
// The expires b-tree formula
|
||||
@@ -1295,6 +1441,12 @@ func (dbi *dbItem) Less(item btree.Item, ctx interface{}) bool {
|
||||
return dbi.key < dbi2.key
|
||||
}
|
||||
|
||||
func lessCtx(ctx interface{}) func(a, b interface{}) bool {
|
||||
return func(a, b interface{}) bool {
|
||||
return a.(*dbItem).Less(b.(*dbItem), ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Rect converts a string to a rectangle.
|
||||
// An invalid rectangle will cause a panic.
|
||||
func (dbi *dbItem) Rect(ctx interface{}) (min, max []float64) {
|
||||
@@ -1384,7 +1536,9 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string,
|
||||
// create a rollback entry with a nil value. A nil value indicates
|
||||
// that the entry should be deleted on rollback. When the value is
|
||||
// *not* nil, that means the entry should be reverted.
|
||||
tx.wc.rollbackItems[key] = nil
|
||||
if _, ok := tx.wc.rollbackItems[key]; !ok {
|
||||
tx.wc.rollbackItems[key] = nil
|
||||
}
|
||||
} else {
|
||||
// A previous item already exists in the database. Let's create a
|
||||
// rollback entry with the item as the value. We need to check the
|
||||
@@ -1475,7 +1629,7 @@ func (tx *Tx) TTL(key string) (time.Duration, error) {
|
||||
} else if item.opts == nil || !item.opts.ex {
|
||||
return -1, nil
|
||||
}
|
||||
dur := item.opts.exat.Sub(time.Now())
|
||||
dur := time.Until(item.opts.exat)
|
||||
if dur < 0 {
|
||||
return 0, ErrNotFound
|
||||
}
|
||||
@@ -1498,8 +1652,11 @@ func (tx *Tx) scan(desc, gt, lt bool, index, start, stop string,
|
||||
return ErrTxClosed
|
||||
}
|
||||
// wrap a btree specific iterator around the user-defined iterator.
|
||||
iter := func(item btree.Item) bool {
|
||||
iter := func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
if dbi.expired() {
|
||||
return true
|
||||
}
|
||||
return iterator(dbi.key, dbi.val)
|
||||
}
|
||||
var tr *btree.BTree
|
||||
@@ -1542,26 +1699,26 @@ func (tx *Tx) scan(desc, gt, lt bool, index, start, stop string,
|
||||
if desc {
|
||||
if gt {
|
||||
if lt {
|
||||
tr.DescendRange(itemA, itemB, iter)
|
||||
btreeDescendRange(tr, itemA, itemB, iter)
|
||||
} else {
|
||||
tr.DescendGreaterThan(itemA, iter)
|
||||
btreeDescendGreaterThan(tr, itemA, iter)
|
||||
}
|
||||
} else if lt {
|
||||
tr.DescendLessOrEqual(itemA, iter)
|
||||
btreeDescendLessOrEqual(tr, itemA, iter)
|
||||
} else {
|
||||
tr.Descend(iter)
|
||||
btreeDescend(tr, iter)
|
||||
}
|
||||
} else {
|
||||
if gt {
|
||||
if lt {
|
||||
tr.AscendRange(itemA, itemB, iter)
|
||||
btreeAscendRange(tr, itemA, itemB, iter)
|
||||
} else {
|
||||
tr.AscendGreaterOrEqual(itemA, iter)
|
||||
btreeAscendGreaterOrEqual(tr, itemA, iter)
|
||||
}
|
||||
} else if lt {
|
||||
tr.AscendLessThan(itemA, iter)
|
||||
btreeAscendLessThan(tr, itemA, iter)
|
||||
} else {
|
||||
tr.Ascend(iter)
|
||||
btreeAscend(tr, iter)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1816,7 +1973,7 @@ func (tx *Tx) Nearby(index, bounds string,
|
||||
return nil
|
||||
}
|
||||
// // wrap a rtree specific iterator around the user-defined iterator.
|
||||
iter := func(item rtree.Item, dist float64) bool {
|
||||
iter := func(item rtred.Item, dist float64) bool {
|
||||
dbi := item.(*dbItem)
|
||||
return iterator(dbi.key, dbi.val, dist)
|
||||
}
|
||||
@@ -1854,7 +2011,7 @@ func (tx *Tx) Intersects(index, bounds string,
|
||||
return nil
|
||||
}
|
||||
// wrap a rtree specific iterator around the user-defined iterator.
|
||||
iter := func(item rtree.Item) bool {
|
||||
iter := func(item rtred.Item) bool {
|
||||
dbi := item.(*dbItem)
|
||||
return iterator(dbi.key, dbi.val)
|
||||
}
|
||||
@@ -2014,7 +2171,8 @@ func (tx *Tx) createIndex(name string, pattern string,
|
||||
if tx.wc.rbkeys == nil {
|
||||
// store the index in the rollback map.
|
||||
if _, ok := tx.wc.rollbackIndexes[name]; !ok {
|
||||
// we use nil to indicate that the index should be removed upon rollback.
|
||||
// we use nil to indicate that the index should be removed upon
|
||||
// rollback.
|
||||
tx.wc.rollbackIndexes[name] = nil
|
||||
}
|
||||
}
|
||||
@@ -2044,8 +2202,8 @@ func (tx *Tx) DropIndex(name string) error {
|
||||
if tx.wc.rbkeys == nil {
|
||||
// store the index in the rollback map.
|
||||
if _, ok := tx.wc.rollbackIndexes[name]; !ok {
|
||||
// we use a non-nil copy of the index without the data to indicate that the
|
||||
// index should be rebuilt upon rollback.
|
||||
// we use a non-nil copy of the index without the data to indicate
|
||||
// that the index should be rebuilt upon rollback.
|
||||
tx.wc.rollbackIndexes[name] = idx.clearCopy()
|
||||
}
|
||||
}
|
||||
@@ -2181,3 +2339,72 @@ func IndexJSONCaseSensitive(path string) func(a, b string) bool {
|
||||
func Desc(less func(a, b string) bool) func(a, b string) bool {
|
||||
return func(a, b string) bool { return less(b, a) }
|
||||
}
|
||||
|
||||
//// Wrappers around btree Ascend/Descend
|
||||
|
||||
func bLT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(a, b) }
|
||||
func bGT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(b, a) }
|
||||
|
||||
// func bLTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(b, a) }
|
||||
// func bGTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(a, b) }
|
||||
|
||||
// Ascend
|
||||
|
||||
func btreeAscend(tr *btree.BTree, iter func(item interface{}) bool) {
|
||||
tr.Ascend(nil, iter)
|
||||
}
|
||||
|
||||
func btreeAscendLessThan(tr *btree.BTree, pivot interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Ascend(nil, func(item interface{}) bool {
|
||||
return bLT(tr, item, pivot) && iter(item)
|
||||
})
|
||||
}
|
||||
|
||||
func btreeAscendGreaterOrEqual(tr *btree.BTree, pivot interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Ascend(pivot, iter)
|
||||
}
|
||||
|
||||
func btreeAscendRange(tr *btree.BTree, greaterOrEqual, lessThan interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Ascend(greaterOrEqual, func(item interface{}) bool {
|
||||
return bLT(tr, item, lessThan) && iter(item)
|
||||
})
|
||||
}
|
||||
|
||||
// Descend
|
||||
|
||||
func btreeDescend(tr *btree.BTree, iter func(item interface{}) bool) {
|
||||
tr.Descend(nil, iter)
|
||||
}
|
||||
|
||||
func btreeDescendGreaterThan(tr *btree.BTree, pivot interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Descend(nil, func(item interface{}) bool {
|
||||
return bGT(tr, item, pivot) && iter(item)
|
||||
})
|
||||
}
|
||||
|
||||
func btreeDescendRange(tr *btree.BTree, lessOrEqual, greaterThan interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Descend(lessOrEqual, func(item interface{}) bool {
|
||||
return bGT(tr, item, greaterThan) && iter(item)
|
||||
})
|
||||
}
|
||||
|
||||
func btreeDescendLessOrEqual(tr *btree.BTree, pivot interface{},
|
||||
iter func(item interface{}) bool,
|
||||
) {
|
||||
tr.Descend(pivot, iter)
|
||||
}
|
||||
|
||||
func btreeNew(less func(a, b interface{}) bool) *btree.BTree {
|
||||
// Using NewNonConcurrent because we're managing our own locks.
|
||||
return btree.NewNonConcurrent(less)
|
||||
}
|
||||
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
module github.com/tidwall/buntdb
|
||||
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274
|
||||
github.com/tidwall/gjson v1.3.4
|
||||
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
|
||||
github.com/tidwall/match v1.0.1
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e
|
||||
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
|
||||
)
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E=
|
||||
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
|
||||
github.com/tidwall/gjson v1.3.4 h1:On5waDnyKKk3SWE4EthbjjirAWXp43xx5cKCUZY1eZw=
|
||||
github.com/tidwall/gjson v1.3.4/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
|
||||
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
|
||||
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
|
||||
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
|
||||
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
|
||||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
|
||||
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
|
||||
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
|
||||
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
+38
-32
@@ -3,13 +3,12 @@
|
||||
src="logo.png"
|
||||
width="240" height="78" border="0" alt="GJSON">
|
||||
<br>
|
||||
<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
|
||||
<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||
<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
|
||||
<a href="https://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
|
||||
<a href="SYNTAX.md"><img src="https://img.shields.io/badge/{}-syntax-33aa33.svg?style=flat-square" alt="GJSON Syntax"></a>
|
||||
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
<p align="center">get json values quickly</a></p>
|
||||
|
||||
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
|
||||
@@ -17,6 +16,10 @@ It has features such as [one line retrieval](#get-a-value), [dot notation paths]
|
||||
|
||||
Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.
|
||||
|
||||
This README is a quick overview of how to use GJSON, for more information check out [GJSON Syntax](SYNTAX.md).
|
||||
|
||||
GJSON is also available for [Python](https://github.com/volans-/gjson-py) and [Rust](https://github.com/tidwall/gjson.rs)
|
||||
|
||||
Getting Started
|
||||
===============
|
||||
|
||||
@@ -126,11 +129,12 @@ nil, for JSON null
|
||||
To directly access the value:
|
||||
|
||||
```go
|
||||
result.Type // can be String, Number, True, False, Null, or JSON
|
||||
result.Str // holds the string
|
||||
result.Num // holds the float64 number
|
||||
result.Raw // holds the raw json
|
||||
result.Index // index of raw value in original json, zero means index unknown
|
||||
result.Type // can be String, Number, True, False, Null, or JSON
|
||||
result.Str // holds the string
|
||||
result.Num // holds the float64 number
|
||||
result.Raw // holds the raw json
|
||||
result.Index // index of raw value in original json, zero means index unknown
|
||||
result.Indexes // indexes of all the elements that match on a path containing the '#' query character.
|
||||
```
|
||||
|
||||
There are a variety of handy functions that work on a result:
|
||||
@@ -153,10 +157,6 @@ result.Less(token Result, caseSensitive bool) bool
|
||||
|
||||
The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:
|
||||
|
||||
The `result.Array()` function returns back an array of values.
|
||||
If the result represents a non-existent value, then an empty array will be returned.
|
||||
If the result is not a JSON array, the return value will be an array containing one result.
|
||||
|
||||
```go
|
||||
boolean >> bool
|
||||
number >> float64
|
||||
@@ -166,6 +166,10 @@ array >> []interface{}
|
||||
object >> map[string]interface{}
|
||||
```
|
||||
|
||||
The `result.Array()` function returns back an array of values.
|
||||
If the result represents a non-existent value, then an empty array will be returned.
|
||||
If the result is not a JSON array, the return value will be an array containing one result.
|
||||
|
||||
### 64-bit integers
|
||||
|
||||
The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers.
|
||||
@@ -193,11 +197,20 @@ we'll get `children` array and reverse the order:
|
||||
"children|@reverse|0" >> "Jack"
|
||||
```
|
||||
|
||||
There are currently three built-in modifiers:
|
||||
There are currently the following built-in modifiers:
|
||||
|
||||
- `@reverse`: Reverse an array or the members of an object.
|
||||
- `@ugly`: Remove all whitespace from a json document.
|
||||
- `@pretty`: Make the json document more human readable.
|
||||
- `@this`: Returns the current element. It can be used to retrieve the root element.
|
||||
- `@valid`: Ensure the json document is valid.
|
||||
- `@flatten`: Flattens an array.
|
||||
- `@join`: Joins multiple objects into a single object.
|
||||
- `@keys`: Returns an array of keys for an object.
|
||||
- `@values`: Returns an array of values for an object.
|
||||
- `@tostr`: Converts json to a string. Wraps a json string.
|
||||
- `@fromstr`: Converts a string from json. Unwraps a json string.
|
||||
- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db).
|
||||
|
||||
### Modifier arguments
|
||||
|
||||
@@ -432,14 +445,15 @@ Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/js
|
||||
and [json-iterator](https://github.com/json-iterator/go)
|
||||
|
||||
```
|
||||
BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op
|
||||
BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op
|
||||
BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op
|
||||
BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op
|
||||
BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op
|
||||
BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op
|
||||
BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op
|
||||
BenchmarkGJSONGet-16 11644512 311 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkGJSONUnmarshalMap-16 1122678 3094 ns/op 1920 B/op 26 allocs/op
|
||||
BenchmarkJSONUnmarshalMap-16 516681 6810 ns/op 2944 B/op 69 allocs/op
|
||||
BenchmarkJSONUnmarshalStruct-16 697053 5400 ns/op 928 B/op 13 allocs/op
|
||||
BenchmarkJSONDecoder-16 330450 10217 ns/op 3845 B/op 160 allocs/op
|
||||
BenchmarkFFJSONLexer-16 1424979 2585 ns/op 880 B/op 8 allocs/op
|
||||
BenchmarkEasyJSONLexer-16 3000000 729 ns/op 501 B/op 5 allocs/op
|
||||
BenchmarkJSONParserGet-16 3000000 366 ns/op 21 B/op 0 allocs/op
|
||||
BenchmarkJSONIterator-16 3000000 869 ns/op 693 B/op 14 allocs/op
|
||||
```
|
||||
|
||||
JSON document used:
|
||||
@@ -472,7 +486,7 @@ JSON document used:
|
||||
}
|
||||
```
|
||||
|
||||
Each operation was rotated though one of the following search paths:
|
||||
Each operation was rotated through one of the following search paths:
|
||||
|
||||
```
|
||||
widget.window.name
|
||||
@@ -480,12 +494,4 @@ widget.image.hOffset
|
||||
widget.text.onMouseUp
|
||||
```
|
||||
|
||||
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
|
||||
|
||||
|
||||
## Contact
|
||||
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||
|
||||
## License
|
||||
|
||||
GJSON source code is available under the MIT [License](/LICENSE).
|
||||
*These benchmarks were run on a MacBook Pro 16" 2.4 GHz Intel Core i9 using Go 1.17 and can be found [here](https://github.com/tidwall/gjson-benchmarks).*
|
||||
|
||||
+84
-7
@@ -13,16 +13,16 @@ This document is designed to explain the structure of a GJSON Path through examp
|
||||
- [Dot vs Pipe](#dot-vs-pipe)
|
||||
- [Modifiers](#modifiers)
|
||||
- [Multipaths](#multipaths)
|
||||
- [Literals](#literals)
|
||||
|
||||
The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson).
|
||||
Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online.
|
||||
|
||||
|
||||
## Path structure
|
||||
|
||||
A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character.
|
||||
|
||||
Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, and `?`.
|
||||
Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, `!`, and `?`.
|
||||
|
||||
## Example
|
||||
|
||||
@@ -77,6 +77,21 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`.
|
||||
fav\.movie "Deer Hunter"
|
||||
```
|
||||
|
||||
You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in your source code.
|
||||
|
||||
```go
|
||||
// Go
|
||||
val := gjson.Get(json, "fav\\.movie") // must escape the slash
|
||||
val := gjson.Get(json, `fav\.movie`) // no need to escape the slash
|
||||
```
|
||||
|
||||
```rust
|
||||
// Rust
|
||||
let val = gjson::get(json, "fav\\.movie") // must escape the slash
|
||||
let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash
|
||||
```
|
||||
|
||||
|
||||
### Arrays
|
||||
|
||||
The `#` character allows for digging into JSON Arrays.
|
||||
@@ -120,6 +135,37 @@ changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths)
|
||||
syntax. For backwards compatibility, `#[...]` will continue to work until the
|
||||
next major release.*
|
||||
|
||||
The `~` (tilde) operator will convert a value to a boolean before comparison.
|
||||
|
||||
For example, using the following JSON:
|
||||
|
||||
```json
|
||||
{
|
||||
"vals": [
|
||||
{ "a": 1, "b": true },
|
||||
{ "a": 2, "b": true },
|
||||
{ "a": 3, "b": false },
|
||||
{ "a": 4, "b": "0" },
|
||||
{ "a": 5, "b": 0 },
|
||||
{ "a": 6, "b": "1" },
|
||||
{ "a": 7, "b": 1 },
|
||||
{ "a": 8, "b": "true" },
|
||||
{ "a": 9, "b": false },
|
||||
{ "a": 10, "b": null },
|
||||
{ "a": 11 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
You can now query for all true(ish) or false(ish) values:
|
||||
|
||||
```
|
||||
vals.#(b==~true)#.a >> [1,2,6,7,8]
|
||||
vals.#(b==~false)#.a >> [3,4,5,9,10,11]
|
||||
```
|
||||
|
||||
The last value which was non-existent is treated as `false`
|
||||
|
||||
### Dot vs Pipe
|
||||
|
||||
The `.` is standard separator, but it's also possible to use a `|`.
|
||||
@@ -181,11 +227,20 @@ children.@reverse ["Jack","Alex","Sara"]
|
||||
children.@reverse.0 "Jack"
|
||||
```
|
||||
|
||||
There are currently three built-in modifiers:
|
||||
There are currently the following built-in modifiers:
|
||||
|
||||
- `@reverse`: Reverse an array or the members of an object.
|
||||
- `@ugly`: Remove all whitespace from JSON.
|
||||
- `@pretty`: Make the JSON more human readable.
|
||||
- `@this`: Returns the current element. It can be used to retrieve the root element.
|
||||
- `@valid`: Ensure the json document is valid.
|
||||
- `@flatten`: Flattens an array.
|
||||
- `@join`: Joins multiple objects into a single object.
|
||||
- `@keys`: Returns an array of keys for an object.
|
||||
- `@values`: Returns an array of values for an object.
|
||||
- `@tostr`: Converts json to a string. Wraps a json string.
|
||||
- `@fromstr`: Converts a string from json. Unwraps a json string.
|
||||
- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db).
|
||||
|
||||
#### Modifier arguments
|
||||
|
||||
@@ -236,13 +291,15 @@ gjson.AddModifier("case", func(json, arg string) string {
|
||||
"children.@case:lower.@reverse" ["jack","alex","sara"]
|
||||
```
|
||||
|
||||
*Note: Custom modifiers are not yet available in the Rust version*
|
||||
|
||||
### Multipaths
|
||||
|
||||
Starting with v1.3.0, GJSON added the ability to join multiple paths together
|
||||
to form new documents. Wrapping comma-separated paths between `{...}` or
|
||||
`[...]` will result in a new array or object, respectively.
|
||||
to form new documents. Wrapping comma-separated paths between `[...]` or
|
||||
`{...}` will result in a new array or object, respectively.
|
||||
|
||||
For example, using the given multipath
|
||||
For example, using the given multipath:
|
||||
|
||||
```
|
||||
{name.first,age,"the_murphys":friends.#(last="Murphy")#.first}
|
||||
@@ -258,8 +315,28 @@ determined, then "_" is used.
|
||||
|
||||
This results in
|
||||
|
||||
```
|
||||
```json
|
||||
{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]}
|
||||
```
|
||||
|
||||
### Literals
|
||||
|
||||
Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths).
|
||||
|
||||
A json literal begins with the '!' declaration character.
|
||||
|
||||
For example, using the given multipath:
|
||||
|
||||
```
|
||||
{name.first,age,"company":!"Happysoft","employed":!true}
|
||||
```
|
||||
|
||||
Here we selected the first name and age. Then add two new fields, "company" and "employed".
|
||||
|
||||
This results in
|
||||
|
||||
```json
|
||||
{"first":"Tom","age":37,"company":"Happysoft","employed":true}
|
||||
```
|
||||
|
||||
*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.*
|
||||
|
||||
+964
-431
File diff suppressed because it is too large
Load Diff
-18
@@ -1,18 +0,0 @@
|
||||
//+build appengine js
|
||||
|
||||
package gjson
|
||||
|
||||
func getBytes(json []byte, path string) Result {
|
||||
return Get(string(json), path)
|
||||
}
|
||||
func fillIndex(json string, c *parseContext) {
|
||||
// noop. Use zero for the Index value.
|
||||
}
|
||||
|
||||
func stringBytes(s string) []byte {
|
||||
return []byte(s)
|
||||
}
|
||||
|
||||
func bytesString(b []byte) string {
|
||||
return string(b)
|
||||
}
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
//+build !appengine
|
||||
//+build !js
|
||||
|
||||
package gjson
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// getBytes casts the input json bytes to a string and safely returns the
|
||||
// results as uniquely allocated data. This operation is intended to minimize
|
||||
// copies and allocations for the large json string->[]byte.
|
||||
func getBytes(json []byte, path string) Result {
|
||||
var result Result
|
||||
if json != nil {
|
||||
// unsafe cast to string
|
||||
result = Get(*(*string)(unsafe.Pointer(&json)), path)
|
||||
// safely get the string headers
|
||||
rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
|
||||
strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
|
||||
// create byte slice headers
|
||||
rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
|
||||
strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
|
||||
if strh.Data == 0 {
|
||||
// str is nil
|
||||
if rawh.Data == 0 {
|
||||
// raw is nil
|
||||
result.Raw = ""
|
||||
} else {
|
||||
// raw has data, safely copy the slice header to a string
|
||||
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
|
||||
}
|
||||
result.Str = ""
|
||||
} else if rawh.Data == 0 {
|
||||
// raw is nil
|
||||
result.Raw = ""
|
||||
// str has data, safely copy the slice header to a string
|
||||
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
|
||||
} else if strh.Data >= rawh.Data &&
|
||||
int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
|
||||
// Str is a substring of Raw.
|
||||
start := int(strh.Data - rawh.Data)
|
||||
// safely copy the raw slice header
|
||||
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
|
||||
// substring the raw
|
||||
result.Str = result.Raw[start : start+strh.Len]
|
||||
} else {
|
||||
// safely copy both the raw and str slice headers to strings
|
||||
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
|
||||
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fillIndex finds the position of Raw data and assigns it to the Index field
|
||||
// of the resulting value. If the position cannot be found then Index zero is
|
||||
// used instead.
|
||||
func fillIndex(json string, c *parseContext) {
|
||||
if len(c.value.Raw) > 0 && !c.calcd {
|
||||
jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
|
||||
rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
|
||||
c.value.Index = int(rhdr.Data - jhdr.Data)
|
||||
if c.value.Index < 0 || c.value.Index >= len(json) {
|
||||
c.value.Index = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stringBytes(s string) []byte {
|
||||
return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
|
||||
Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data,
|
||||
Len: len(s),
|
||||
Cap: len(s),
|
||||
}))
|
||||
}
|
||||
|
||||
func bytesString(b []byte) string {
|
||||
return *(*string)(unsafe.Pointer(&b))
|
||||
}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
module github.com/tidwall/gjson
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/tidwall/match v1.0.1
|
||||
github.com/tidwall/pretty v1.0.0
|
||||
)
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
|
||||
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
|
||||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
+9
-12
@@ -1,20 +1,17 @@
|
||||
Match
|
||||
=====
|
||||
<a href="https://travis-ci.org/tidwall/match"><img src="https://img.shields.io/travis/tidwall/match.svg?style=flat-square" alt="Build Status"></a>
|
||||
<a href="https://godoc.org/github.com/tidwall/match"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
|
||||
# Match
|
||||
|
||||
[](https://godoc.org/github.com/tidwall/match)
|
||||
|
||||
Match is a very simple pattern matcher where '*' matches on any
|
||||
number characters and '?' matches on any one character.
|
||||
|
||||
Installing
|
||||
----------
|
||||
## Installing
|
||||
|
||||
```
|
||||
go get -u github.com/tidwall/match
|
||||
```
|
||||
|
||||
Example
|
||||
-------
|
||||
## Example
|
||||
|
||||
```go
|
||||
match.Match("hello", "*llo")
|
||||
@@ -23,10 +20,10 @@ match.Match("hello", "h*o")
|
||||
```
|
||||
|
||||
|
||||
Contact
|
||||
-------
|
||||
## Contact
|
||||
|
||||
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||
|
||||
License
|
||||
-------
|
||||
## License
|
||||
|
||||
Redcon source code is available under the MIT [License](/LICENSE).
|
||||
|
||||
+153
-97
@@ -1,12 +1,14 @@
|
||||
// Match provides a simple pattern matcher with unicode support.
|
||||
// Package match provides a simple pattern matcher with unicode support.
|
||||
package match
|
||||
|
||||
import "unicode/utf8"
|
||||
import (
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Match returns true if str matches pattern. This is a very
|
||||
// simple wildcard match where '*' matches on any number characters
|
||||
// and '?' matches on any one character.
|
||||
|
||||
//
|
||||
// pattern:
|
||||
// { term }
|
||||
// term:
|
||||
@@ -19,113 +21,167 @@ func Match(str, pattern string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
return deepMatch(str, pattern)
|
||||
}
|
||||
func deepMatch(str, pattern string) bool {
|
||||
for len(pattern) > 0 {
|
||||
if pattern[0] > 0x7f {
|
||||
return deepMatchRune(str, pattern)
|
||||
}
|
||||
switch pattern[0] {
|
||||
default:
|
||||
if len(str) == 0 {
|
||||
return false
|
||||
}
|
||||
if str[0] > 0x7f {
|
||||
return deepMatchRune(str, pattern)
|
||||
}
|
||||
if str[0] != pattern[0] {
|
||||
return false
|
||||
}
|
||||
case '?':
|
||||
if len(str) == 0 {
|
||||
return false
|
||||
}
|
||||
case '*':
|
||||
return deepMatch(str, pattern[1:]) ||
|
||||
(len(str) > 0 && deepMatch(str[1:], pattern))
|
||||
}
|
||||
str = str[1:]
|
||||
pattern = pattern[1:]
|
||||
}
|
||||
return len(str) == 0 && len(pattern) == 0
|
||||
return match(str, pattern, 0, nil, -1) == rMatch
|
||||
}
|
||||
|
||||
func deepMatchRune(str, pattern string) bool {
|
||||
var sr, pr rune
|
||||
var srsz, prsz int
|
||||
// MatchLimit is the same as Match but will limit the complexity of the match
|
||||
// operation. This is to avoid long running matches, specifically to avoid ReDos
|
||||
// attacks from arbritary inputs.
|
||||
//
|
||||
// How it works:
|
||||
// The underlying match routine is recursive and may call itself when it
|
||||
// encounters a sandwiched wildcard pattern, such as: `user:*:name`.
|
||||
// Everytime it calls itself a counter is incremented.
|
||||
// The operation is stopped when counter > maxcomp*len(str).
|
||||
func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) {
|
||||
if pattern == "*" {
|
||||
return true, false
|
||||
}
|
||||
counter := 0
|
||||
r := match(str, pattern, len(str), &counter, maxcomp)
|
||||
if r == rStop {
|
||||
return false, true
|
||||
}
|
||||
return r == rMatch, false
|
||||
}
|
||||
|
||||
// read the first rune ahead of time
|
||||
if len(str) > 0 {
|
||||
if str[0] > 0x7f {
|
||||
sr, srsz = utf8.DecodeRuneInString(str)
|
||||
} else {
|
||||
sr, srsz = rune(str[0]), 1
|
||||
type result int
|
||||
|
||||
const (
|
||||
rNoMatch result = iota
|
||||
rMatch
|
||||
rStop
|
||||
)
|
||||
|
||||
func match(str, pat string, slen int, counter *int, maxcomp int) result {
|
||||
// check complexity limit
|
||||
if maxcomp > -1 {
|
||||
if *counter > slen*maxcomp {
|
||||
return rStop
|
||||
}
|
||||
} else {
|
||||
sr, srsz = utf8.RuneError, 0
|
||||
*counter++
|
||||
}
|
||||
if len(pattern) > 0 {
|
||||
if pattern[0] > 0x7f {
|
||||
pr, prsz = utf8.DecodeRuneInString(pattern)
|
||||
} else {
|
||||
pr, prsz = rune(pattern[0]), 1
|
||||
|
||||
for len(pat) > 0 {
|
||||
var wild bool
|
||||
pc, ps := rune(pat[0]), 1
|
||||
if pc > 0x7f {
|
||||
pc, ps = utf8.DecodeRuneInString(pat)
|
||||
}
|
||||
} else {
|
||||
pr, prsz = utf8.RuneError, 0
|
||||
}
|
||||
// done reading
|
||||
for pr != utf8.RuneError {
|
||||
switch pr {
|
||||
default:
|
||||
if srsz == utf8.RuneError {
|
||||
return false
|
||||
}
|
||||
if sr != pr {
|
||||
return false
|
||||
}
|
||||
case '?':
|
||||
if srsz == utf8.RuneError {
|
||||
return false
|
||||
}
|
||||
case '*':
|
||||
return deepMatchRune(str, pattern[prsz:]) ||
|
||||
(srsz > 0 && deepMatchRune(str[srsz:], pattern))
|
||||
}
|
||||
str = str[srsz:]
|
||||
pattern = pattern[prsz:]
|
||||
// read the next runes
|
||||
var sc rune
|
||||
var ss int
|
||||
if len(str) > 0 {
|
||||
if str[0] > 0x7f {
|
||||
sr, srsz = utf8.DecodeRuneInString(str)
|
||||
} else {
|
||||
sr, srsz = rune(str[0]), 1
|
||||
sc, ss = rune(str[0]), 1
|
||||
if sc > 0x7f {
|
||||
sc, ss = utf8.DecodeRuneInString(str)
|
||||
}
|
||||
} else {
|
||||
sr, srsz = utf8.RuneError, 0
|
||||
}
|
||||
if len(pattern) > 0 {
|
||||
if pattern[0] > 0x7f {
|
||||
pr, prsz = utf8.DecodeRuneInString(pattern)
|
||||
} else {
|
||||
pr, prsz = rune(pattern[0]), 1
|
||||
switch pc {
|
||||
case '?':
|
||||
if ss == 0 {
|
||||
return rNoMatch
|
||||
}
|
||||
case '*':
|
||||
// Ignore repeating stars.
|
||||
for len(pat) > 1 && pat[1] == '*' {
|
||||
pat = pat[1:]
|
||||
}
|
||||
} else {
|
||||
pr, prsz = utf8.RuneError, 0
|
||||
}
|
||||
// done reading
|
||||
}
|
||||
|
||||
return srsz == 0 && prsz == 0
|
||||
// If this star is the last character then it must be a match.
|
||||
if len(pat) == 1 {
|
||||
return rMatch
|
||||
}
|
||||
|
||||
// Match and trim any non-wildcard suffix characters.
|
||||
var ok bool
|
||||
str, pat, ok = matchTrimSuffix(str, pat)
|
||||
if !ok {
|
||||
return rNoMatch
|
||||
}
|
||||
|
||||
// Check for single star again.
|
||||
if len(pat) == 1 {
|
||||
return rMatch
|
||||
}
|
||||
|
||||
// Perform recursive wildcard search.
|
||||
r := match(str, pat[1:], slen, counter, maxcomp)
|
||||
if r != rNoMatch {
|
||||
return r
|
||||
}
|
||||
if len(str) == 0 {
|
||||
return rNoMatch
|
||||
}
|
||||
wild = true
|
||||
default:
|
||||
if ss == 0 {
|
||||
return rNoMatch
|
||||
}
|
||||
if pc == '\\' {
|
||||
pat = pat[ps:]
|
||||
pc, ps = utf8.DecodeRuneInString(pat)
|
||||
if ps == 0 {
|
||||
return rNoMatch
|
||||
}
|
||||
}
|
||||
if sc != pc {
|
||||
return rNoMatch
|
||||
}
|
||||
}
|
||||
str = str[ss:]
|
||||
if !wild {
|
||||
pat = pat[ps:]
|
||||
}
|
||||
}
|
||||
if len(str) == 0 {
|
||||
return rMatch
|
||||
}
|
||||
return rNoMatch
|
||||
}
|
||||
|
||||
var maxRuneBytes = func() []byte {
|
||||
b := make([]byte, 4)
|
||||
if utf8.EncodeRune(b, '\U0010FFFF') != 4 {
|
||||
panic("invalid rune encoding")
|
||||
// matchTrimSuffix matches and trims any non-wildcard suffix characters.
|
||||
// Returns the trimed string and pattern.
|
||||
//
|
||||
// This is called because the pattern contains extra data after the wildcard
|
||||
// star. Here we compare any suffix characters in the pattern to the suffix of
|
||||
// the target string. Basically a reverse match that stops when a wildcard
|
||||
// character is reached. This is a little trickier than a forward match because
|
||||
// we need to evaluate an escaped character in reverse.
|
||||
//
|
||||
// Any matched characters will be trimmed from both the target
|
||||
// string and the pattern.
|
||||
func matchTrimSuffix(str, pat string) (string, string, bool) {
|
||||
// It's expected that the pattern has at least two bytes and the first byte
|
||||
// is a wildcard star '*'
|
||||
match := true
|
||||
for len(str) > 0 && len(pat) > 1 {
|
||||
pc, ps := utf8.DecodeLastRuneInString(pat)
|
||||
var esc bool
|
||||
for i := 0; ; i++ {
|
||||
if pat[len(pat)-ps-i-1] != '\\' {
|
||||
if i&1 == 1 {
|
||||
esc = true
|
||||
ps++
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if pc == '*' && !esc {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
sc, ss := utf8.DecodeLastRuneInString(str)
|
||||
if !((pc == '?' && !esc) || pc == sc) {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
str = str[:len(str)-ss]
|
||||
pat = pat[:len(pat)-ps]
|
||||
}
|
||||
return b
|
||||
}()
|
||||
return str, pat, match
|
||||
}
|
||||
|
||||
var maxRuneBytes = [...]byte{244, 143, 191, 191}
|
||||
|
||||
// Allowable parses the pattern and determines the minimum and maximum allowable
|
||||
// values that the pattern can represent.
|
||||
@@ -146,7 +202,7 @@ func Allowable(pattern string) (min, max string) {
|
||||
}
|
||||
if pattern[i] == '?' {
|
||||
minb = append(minb, 0)
|
||||
maxb = append(maxb, maxRuneBytes...)
|
||||
maxb = append(maxb, maxRuneBytes[:]...)
|
||||
} else {
|
||||
minb = append(minb, pattern[i])
|
||||
maxb = append(maxb, pattern[i])
|
||||
|
||||
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
+8
-10
@@ -1,8 +1,6 @@
|
||||
# Pretty
|
||||
[](https://travis-ci.org/tidwall/prettty)
|
||||
[](http://gocover.io/github.com/tidwall/pretty)
|
||||
[](https://godoc.org/github.com/tidwall/pretty)
|
||||
|
||||
[](https://pkg.go.dev/github.com/tidwall/pretty)
|
||||
|
||||
Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.
|
||||
|
||||
@@ -81,7 +79,6 @@ Will format the json to:
|
||||
{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
|
||||
```
|
||||
|
||||
|
||||
## Customized output
|
||||
|
||||
There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options:
|
||||
@@ -106,14 +103,15 @@ type Options struct {
|
||||
|
||||
Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods.
|
||||
```
|
||||
BenchmarkPretty-8 1000000 1283 ns/op 720 B/op 2 allocs/op
|
||||
BenchmarkUgly-8 3000000 426 ns/op 240 B/op 1 allocs/op
|
||||
BenchmarkUglyInPlace-8 5000000 340 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkJSONIndent-8 300000 4628 ns/op 1069 B/op 4 allocs/op
|
||||
BenchmarkJSONCompact-8 1000000 2469 ns/op 758 B/op 4 allocs/op
|
||||
BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op
|
||||
BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op
|
||||
BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op
|
||||
BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op
|
||||
BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op
|
||||
```
|
||||
|
||||
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.*
|
||||
*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.*
|
||||
|
||||
## Contact
|
||||
Josh Baker [@tidwall](http://twitter.com/tidwall)
|
||||
|
||||
+272
-30
@@ -1,7 +1,10 @@
|
||||
package pretty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Options is Pretty options
|
||||
@@ -84,6 +87,14 @@ func ugly(dst, src []byte) []byte {
|
||||
return dst
|
||||
}
|
||||
|
||||
func isNaNOrInf(src []byte) bool {
|
||||
return src[0] == 'i' || //Inf
|
||||
src[0] == 'I' || // inf
|
||||
src[0] == '+' || // +Inf
|
||||
src[0] == 'N' || // Nan
|
||||
(src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan
|
||||
}
|
||||
|
||||
func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
|
||||
for ; i < len(json); i++ {
|
||||
if json[i] <= ' ' {
|
||||
@@ -92,7 +103,8 @@ func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, in
|
||||
if json[i] == '"' {
|
||||
return appendPrettyString(buf, json, i, nl)
|
||||
}
|
||||
if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' {
|
||||
|
||||
if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) {
|
||||
return appendPrettyNumber(buf, json, i, nl)
|
||||
}
|
||||
if json[i] == '{' {
|
||||
@@ -118,25 +130,121 @@ type pair struct {
|
||||
vstart, vend int
|
||||
}
|
||||
|
||||
type byKey struct {
|
||||
type byKeyVal struct {
|
||||
sorted bool
|
||||
json []byte
|
||||
buf []byte
|
||||
pairs []pair
|
||||
}
|
||||
|
||||
func (arr *byKey) Len() int {
|
||||
func (arr *byKeyVal) Len() int {
|
||||
return len(arr.pairs)
|
||||
}
|
||||
func (arr *byKey) Less(i, j int) bool {
|
||||
key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1]
|
||||
key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1]
|
||||
return string(key1) < string(key2)
|
||||
func (arr *byKeyVal) Less(i, j int) bool {
|
||||
if arr.isLess(i, j, byKey) {
|
||||
return true
|
||||
}
|
||||
if arr.isLess(j, i, byKey) {
|
||||
return false
|
||||
}
|
||||
return arr.isLess(i, j, byVal)
|
||||
}
|
||||
func (arr *byKey) Swap(i, j int) {
|
||||
func (arr *byKeyVal) Swap(i, j int) {
|
||||
arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
|
||||
arr.sorted = true
|
||||
}
|
||||
|
||||
type byKind int
|
||||
|
||||
const (
|
||||
byKey byKind = 0
|
||||
byVal byKind = 1
|
||||
)
|
||||
|
||||
type jtype int
|
||||
|
||||
const (
|
||||
jnull jtype = iota
|
||||
jfalse
|
||||
jnumber
|
||||
jstring
|
||||
jtrue
|
||||
jjson
|
||||
)
|
||||
|
||||
func getjtype(v []byte) jtype {
|
||||
if len(v) == 0 {
|
||||
return jnull
|
||||
}
|
||||
switch v[0] {
|
||||
case '"':
|
||||
return jstring
|
||||
case 'f':
|
||||
return jfalse
|
||||
case 't':
|
||||
return jtrue
|
||||
case 'n':
|
||||
return jnull
|
||||
case '[', '{':
|
||||
return jjson
|
||||
default:
|
||||
return jnumber
|
||||
}
|
||||
}
|
||||
|
||||
func (arr *byKeyVal) isLess(i, j int, kind byKind) bool {
|
||||
k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend]
|
||||
k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend]
|
||||
var v1, v2 []byte
|
||||
if kind == byKey {
|
||||
v1 = k1
|
||||
v2 = k2
|
||||
} else {
|
||||
v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend])
|
||||
v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend])
|
||||
if len(v1) >= len(k1)+1 {
|
||||
v1 = bytes.TrimSpace(v1[len(k1)+1:])
|
||||
}
|
||||
if len(v2) >= len(k2)+1 {
|
||||
v2 = bytes.TrimSpace(v2[len(k2)+1:])
|
||||
}
|
||||
}
|
||||
t1 := getjtype(v1)
|
||||
t2 := getjtype(v2)
|
||||
if t1 < t2 {
|
||||
return true
|
||||
}
|
||||
if t1 > t2 {
|
||||
return false
|
||||
}
|
||||
if t1 == jstring {
|
||||
s1 := parsestr(v1)
|
||||
s2 := parsestr(v2)
|
||||
return string(s1) < string(s2)
|
||||
}
|
||||
if t1 == jnumber {
|
||||
n1, _ := strconv.ParseFloat(string(v1), 64)
|
||||
n2, _ := strconv.ParseFloat(string(v2), 64)
|
||||
return n1 < n2
|
||||
}
|
||||
return string(v1) < string(v2)
|
||||
|
||||
}
|
||||
|
||||
func parsestr(s []byte) []byte {
|
||||
for i := 1; i < len(s); i++ {
|
||||
if s[i] == '\\' {
|
||||
var str string
|
||||
json.Unmarshal(s, &str)
|
||||
return []byte(str)
|
||||
}
|
||||
if s[i] == '"' {
|
||||
return s[1:i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
|
||||
var ok bool
|
||||
if width > 0 {
|
||||
@@ -174,7 +282,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
|
||||
}
|
||||
if n > 0 {
|
||||
nl = len(buf)
|
||||
buf = append(buf, '\n')
|
||||
if buf[nl-1] == ' ' {
|
||||
buf[nl-1] = '\n'
|
||||
} else {
|
||||
buf = append(buf, '\n')
|
||||
}
|
||||
}
|
||||
if buf[len(buf)-1] != open {
|
||||
buf = appendTabs(buf, prefix, indent, tabs)
|
||||
@@ -193,7 +305,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
|
||||
var p pair
|
||||
if pretty {
|
||||
nl = len(buf)
|
||||
buf = append(buf, '\n')
|
||||
if buf[nl-1] == ' ' {
|
||||
buf[nl-1] = '\n'
|
||||
} else {
|
||||
buf = append(buf, '\n')
|
||||
}
|
||||
if open == '{' && sortkeys {
|
||||
p.kstart = i
|
||||
p.vstart = len(buf)
|
||||
@@ -235,8 +351,8 @@ func sortPairs(json, buf []byte, pairs []pair) []byte {
|
||||
}
|
||||
vstart := pairs[0].vstart
|
||||
vend := pairs[len(pairs)-1].vend
|
||||
arr := byKey{false, json, pairs}
|
||||
sort.Sort(&arr)
|
||||
arr := byKeyVal{false, json, buf, pairs}
|
||||
sort.Stable(&arr)
|
||||
if !arr.sorted {
|
||||
return buf
|
||||
}
|
||||
@@ -305,6 +421,7 @@ func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
|
||||
type Style struct {
|
||||
Key, String, Number [2]string
|
||||
True, False, Null [2]string
|
||||
Escape [2]string
|
||||
Append func(dst []byte, c byte) []byte
|
||||
}
|
||||
|
||||
@@ -318,21 +435,26 @@ func hexp(p byte) byte {
|
||||
}
|
||||
|
||||
// TerminalStyle is for terminals
|
||||
var TerminalStyle = &Style{
|
||||
Key: [2]string{"\x1B[94m", "\x1B[0m"},
|
||||
String: [2]string{"\x1B[92m", "\x1B[0m"},
|
||||
Number: [2]string{"\x1B[93m", "\x1B[0m"},
|
||||
True: [2]string{"\x1B[96m", "\x1B[0m"},
|
||||
False: [2]string{"\x1B[96m", "\x1B[0m"},
|
||||
Null: [2]string{"\x1B[91m", "\x1B[0m"},
|
||||
Append: func(dst []byte, c byte) []byte {
|
||||
if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
|
||||
dst = append(dst, "\\u00"...)
|
||||
dst = append(dst, hexp((c>>4)&0xF))
|
||||
return append(dst, hexp((c)&0xF))
|
||||
}
|
||||
return append(dst, c)
|
||||
},
|
||||
var TerminalStyle *Style
|
||||
|
||||
func init() {
|
||||
TerminalStyle = &Style{
|
||||
Key: [2]string{"\x1B[94m", "\x1B[0m"},
|
||||
String: [2]string{"\x1B[92m", "\x1B[0m"},
|
||||
Number: [2]string{"\x1B[93m", "\x1B[0m"},
|
||||
True: [2]string{"\x1B[96m", "\x1B[0m"},
|
||||
False: [2]string{"\x1B[96m", "\x1B[0m"},
|
||||
Null: [2]string{"\x1B[91m", "\x1B[0m"},
|
||||
Escape: [2]string{"\x1B[35m", "\x1B[0m"},
|
||||
Append: func(dst []byte, c byte) []byte {
|
||||
if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
|
||||
dst = append(dst, "\\u00"...)
|
||||
dst = append(dst, hexp((c>>4)&0xF))
|
||||
return append(dst, hexp((c)&0xF))
|
||||
}
|
||||
return append(dst, c)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Color will colorize the json. The style parma is used for customizing
|
||||
@@ -363,8 +485,39 @@ func Color(src []byte, style *Style) []byte {
|
||||
dst = append(dst, style.String[0]...)
|
||||
}
|
||||
dst = apnd(dst, '"')
|
||||
esc := false
|
||||
uesc := 0
|
||||
for i = i + 1; i < len(src); i++ {
|
||||
dst = apnd(dst, src[i])
|
||||
if src[i] == '\\' {
|
||||
if key {
|
||||
dst = append(dst, style.Key[1]...)
|
||||
} else {
|
||||
dst = append(dst, style.String[1]...)
|
||||
}
|
||||
dst = append(dst, style.Escape[0]...)
|
||||
dst = apnd(dst, src[i])
|
||||
esc = true
|
||||
if i+1 < len(src) && src[i+1] == 'u' {
|
||||
uesc = 5
|
||||
} else {
|
||||
uesc = 1
|
||||
}
|
||||
} else if esc {
|
||||
dst = apnd(dst, src[i])
|
||||
if uesc == 1 {
|
||||
esc = false
|
||||
dst = append(dst, style.Escape[1]...)
|
||||
if key {
|
||||
dst = append(dst, style.Key[0]...)
|
||||
} else {
|
||||
dst = append(dst, style.String[0]...)
|
||||
}
|
||||
} else {
|
||||
uesc--
|
||||
}
|
||||
} else {
|
||||
dst = apnd(dst, src[i])
|
||||
}
|
||||
if src[i] == '"' {
|
||||
j := i - 1
|
||||
for ; ; j-- {
|
||||
@@ -377,7 +530,9 @@ func Color(src []byte, style *Style) []byte {
|
||||
}
|
||||
}
|
||||
}
|
||||
if key {
|
||||
if esc {
|
||||
dst = append(dst, style.Escape[1]...)
|
||||
} else if key {
|
||||
dst = append(dst, style.Key[1]...)
|
||||
} else {
|
||||
dst = append(dst, style.String[1]...)
|
||||
@@ -393,7 +548,7 @@ func Color(src []byte, style *Style) []byte {
|
||||
dst = apnd(dst, src[i])
|
||||
} else {
|
||||
var kind byte
|
||||
if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' {
|
||||
if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) {
|
||||
kind = '0'
|
||||
dst = append(dst, style.Number[0]...)
|
||||
} else if src[i] == 't' {
|
||||
@@ -430,3 +585,90 @@ func Color(src []byte, style *Style) []byte {
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// Spec strips out comments and trailing commas and convert the input to a
|
||||
// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
|
||||
//
|
||||
// The resulting JSON will always be the same length as the input and it will
|
||||
// include all of the same line breaks at matching offsets. This is to ensure
|
||||
// the result can be later processed by a external parser and that that
|
||||
// parser will report messages or errors with the correct offsets.
|
||||
func Spec(src []byte) []byte {
|
||||
return spec(src, nil)
|
||||
}
|
||||
|
||||
// SpecInPlace is the same as Spec, but this method reuses the input json
|
||||
// buffer to avoid allocations. Do not use the original bytes slice upon return.
|
||||
func SpecInPlace(src []byte) []byte {
|
||||
return spec(src, src)
|
||||
}
|
||||
|
||||
func spec(src, dst []byte) []byte {
|
||||
dst = dst[:0]
|
||||
for i := 0; i < len(src); i++ {
|
||||
if src[i] == '/' {
|
||||
if i < len(src)-1 {
|
||||
if src[i+1] == '/' {
|
||||
dst = append(dst, ' ', ' ')
|
||||
i += 2
|
||||
for ; i < len(src); i++ {
|
||||
if src[i] == '\n' {
|
||||
dst = append(dst, '\n')
|
||||
break
|
||||
} else if src[i] == '\t' || src[i] == '\r' {
|
||||
dst = append(dst, src[i])
|
||||
} else {
|
||||
dst = append(dst, ' ')
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if src[i+1] == '*' {
|
||||
dst = append(dst, ' ', ' ')
|
||||
i += 2
|
||||
for ; i < len(src)-1; i++ {
|
||||
if src[i] == '*' && src[i+1] == '/' {
|
||||
dst = append(dst, ' ', ' ')
|
||||
i++
|
||||
break
|
||||
} else if src[i] == '\n' || src[i] == '\t' ||
|
||||
src[i] == '\r' {
|
||||
dst = append(dst, src[i])
|
||||
} else {
|
||||
dst = append(dst, ' ')
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
dst = append(dst, src[i])
|
||||
if src[i] == '"' {
|
||||
for i = i + 1; i < len(src); i++ {
|
||||
dst = append(dst, src[i])
|
||||
if src[i] == '"' {
|
||||
j := i - 1
|
||||
for ; ; j-- {
|
||||
if src[j] != '\\' {
|
||||
break
|
||||
}
|
||||
}
|
||||
if (j-i)%2 != 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if src[i] == '}' || src[i] == ']' {
|
||||
for j := len(dst) - 2; j >= 0; j-- {
|
||||
if dst[j] <= ' ' {
|
||||
continue
|
||||
}
|
||||
if dst[j] == ',' {
|
||||
dst[j] = ' '
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
Generated
Vendored
Generated
Vendored
-1
@@ -1,7 +1,6 @@
|
||||
RTree implementation for Go
|
||||
===========================
|
||||
|
||||
[](https://travis-ci.org/tidwall/rtree)
|
||||
[](https://godoc.org/github.com/tidwall/rtree)
|
||||
|
||||
This package provides an in-memory R-Tree implementation for Go, useful as a spatial data structure.
|
||||
Generated
Vendored
Generated
Vendored
Generated
Vendored
+1
@@ -183,6 +183,7 @@ type treeItem struct {
|
||||
item interface{}
|
||||
}
|
||||
|
||||
//go:nocheckptr
|
||||
func (item *treeItem) unsafeNode() *treeNode {
|
||||
return (*treeNode)(unsafe.Pointer(item))
|
||||
}
|
||||
Generated
Vendored
+2
-2
@@ -1,10 +1,10 @@
|
||||
package rtree
|
||||
package rtred
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/tidwall/rtree/base"
|
||||
"github.com/tidwall/rtred/base"
|
||||
)
|
||||
|
||||
type Iterator func(item Item) bool
|
||||
-1
@@ -1 +0,0 @@
|
||||
language: go
|
||||
Reference in New Issue
Block a user