*****************************************************
Summary
-------
Generated at: 2022-09-19T15:38:39+08:00
Notes: 1
Binaries: 2
Archives: 0
Standards: 1112
Apache Licensed: 1112
Generated Documents: 0
JavaDocs are generated, thus a license header is optional.
Generated files do not require license headers.
0 Unknown Licenses
*****************************************************
Files with Apache License headers will be marked AL
Binary files (which do not require any license headers) will be marked B
Compressed archives will be marked A
Notices, licenses etc. will be marked N
AL plasma/pom.xml
AL plasma/test.sh
AL plasma/src/test/resources/logback.xml
AL plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java
AL plasma/src/main/java/org/apache/arrow/plasma/ObjectStoreLink.java
AL plasma/src/main/java/org/apache/arrow/plasma/PlasmaClient.java
AL plasma/src/main/java/org/apache/arrow/plasma/exceptions/DuplicateObjectException.java
AL plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaOutOfMemoryException.java
AL plasma/src/main/java/org/apache/arrow/plasma/exceptions/PlasmaClientException.java
AL plasma/src/main/java/org/apache/arrow/plasma/PlasmaClientJNI.java
AL plasma/src/main/cpp/plasma_client.cc
AL tools/pom.xml
AL tools/src/test/resources/logback.xml
AL tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
AL tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java
AL tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
AL tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
AL tools/src/main/java/org/apache/arrow/tools/Integration.java
AL tools/src/main/java/org/apache/arrow/tools/EchoServer.java
AL tools/src/main/java/org/apache/arrow/tools/FileToStream.java
AL tools/src/main/java/org/apache/arrow/tools/StreamToFile.java
AL tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java
AL memory/memory-unsafe/pom.xml
AL memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java
AL memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java
AL memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
AL memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
AL memory/pom.xml
AL memory/memory-netty/pom.xml
AL memory/memory-netty/src/test/resources/logback.xml
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java
AL memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java
AL memory/memory-netty/src/test/java/io/netty/buffer/TestExpandableByteBuf.java
AL memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java
AL memory/memory-netty/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java
AL memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
AL memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java
AL memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
AL memory/memory-netty/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
AL memory/memory-netty/src/main/java/io/netty/buffer/ExpandableByteBuf.java
AL memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
AL memory/memory-netty/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java
AL memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java
AL memory/memory-netty/src/main/java/io/netty/buffer/LargeBuffer.java
AL memory/memory-core/pom.xml
AL memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/TestAccountant.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestLargeMemoryUtil.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/TestAllocationManager.java
AL memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
AL memory/memory-core/src/test/java/org/apache/arrow/util/TestStackTrace.java
AL memory/memory-core/src/test/java/org/apache/arrow/util/TestCollections2.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/RootAllocator.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/CommonUtil.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/util/StackTrace.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationListener.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcome.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/ReferenceManager.java
AL memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetail
java Arrow Flight sql
需积分: 0 32 浏览量
更新于2022-09-19
收藏 29.83MB ZIP 举报
Java Arrow Flight SQL 是一个在Java平台上处理大数据查询和分析的技术栈,它结合了Apache Arrow和Flight SQL的标准,为高性能的数据处理提供了可能。Apache Arrow是一个跨平台的开源数据层,设计用于加速在内存中的数据处理,而Flight SQL是Apache Arrow项目的一部分,提供了一个通用的SQL查询接口。
Arrow Flight的目标是解决大数据的低延迟传输问题,它允许数据以列式格式快速地在不同的系统之间移动。在Java中,使用Arrow Flight可以极大地提高数据读取和写入的速度,这对于实时分析和大数据处理场景尤其重要。
Java Arrow Flight SQL的实现使得开发者能够利用Java语言的强大功能来编写SQL查询,并利用Arrow的优化特性来提升性能。以下是一些关键知识点:
1. **Apache Arrow格式**:这是一种列式内存格式,用于在计算和存储之间高效地传递数据。它的设计考虑了零拷贝(zero-copy)读取,减少了不必要的数据转换,从而提高了性能。
2. **列式存储**:与传统的行式存储相比,列式存储在处理分析查询时更有效,因为它允许只读取需要的列,减少了I/O操作。
3. **Zero-copy**:Arrow的零拷贝特性意味着数据可以直接在内存中传递,无需先将其复制到临时缓冲区,这降低了CPU开销并提升了速度。
4. **Flight SQL API**:这是Apache Arrow的一部分,定义了一种通用的、基于HTTP/2的协议,用于在客户端和服务器之间传输SQL查询和结果。在Java中,你可以使用这个API来构建SQL查询服务。
5. **Java集成**:Java开发者可以利用Arrow Flight SQL的Java库来构建应用程序,这些库提供了API用于执行SQL查询、处理查询结果,并与其他支持Arrow的数据源交互。
6. **高性能查询**:通过在内存中处理数据和使用高效的列式格式,Arrow Flight SQL可以处理大规模数据集,同时保持低延迟,非常适合实时分析。
7. **兼容性**:由于Apache Arrow和Flight SQL是开源标准,它们与其他编程语言和系统兼容,这使得数据交换变得更加简单,有助于构建多语言和跨平台的数据处理解决方案。
8. **数据类型**:Arrow支持丰富的数据类型,包括固定宽度类型、变量长度类型以及复杂类型如数组和结构,这使得它能处理各种类型的数据。
9. **优化查询执行**:Arrow Flight SQL允许在内存中进行查询优化,比如通过谓词下推(predicate pushdown)、列投影(column projection)等技术,减少不必要的数据处理。
10. **流式处理**:除了单次查询,Arrow Flight还支持流式数据处理,这使得它可以处理无限数据流,例如实时数据流分析。
了解和掌握这些知识点对于开发高效的大数据处理应用至关重要。在Java中,使用Arrow Flight SQL不仅可以提升查询性能,还可以降低开发复杂性,因为SQL是一种广泛使用的查询语言,易于理解和实现。
weixin_42144213
- 粉丝: 0
- 资源: 23