/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.nativeio;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.PerformanceAdvisory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import sun.misc.Unsafe;
import com.google.common.annotations.VisibleForTesting;
/**
* JNI wrappers for various native IO-related calls not available in Java.
* These functions should generally be used alongside a fallback to another
* more portable mechanism.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class NativeIO {
public static class POSIX {
// Flags for open() call from bits/fcntl.h
public static final int O_RDONLY = 00;
public static final int O_WRONLY = 01;
public static final int O_RDWR = 02;
public static final int O_CREAT = 0100;
public static final int O_EXCL = 0200;
public static final int O_NOCTTY = 0400;
public static final int O_TRUNC = 01000;
public static final int O_APPEND = 02000;
public static final int O_NONBLOCK = 04000;
public static final int O_SYNC = 010000;
public static final int O_ASYNC = 020000;
public static final int O_FSYNC = O_SYNC;
public static final int O_NDELAY = O_NONBLOCK;
// Flags for posix_fadvise() from bits/fcntl.h
/* No further special treatment. */
public static final int POSIX_FADV_NORMAL = 0;
/* Expect random page references. */
public static final int POSIX_FADV_RANDOM = 1;
/* Expect sequential page references. */
public static final int POSIX_FADV_SEQUENTIAL = 2;
/* Will need these pages. */
public static final int POSIX_FADV_WILLNEED = 3;
/* Don't need these pages. */
public static final int POSIX_FADV_DONTNEED = 4;
/* Data will be accessed once. */
public static final int POSIX_FADV_NOREUSE = 5;
/* Wait upon writeout of all pages
in the range before performing the
write. */
public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
/* Initiate writeout of all those
dirty pages in the range which are
not presently under writeback. */
public static final int SYNC_FILE_RANGE_WRITE = 2;
/* Wait upon writeout of all pages in
the range after performing the
write. */
public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
private static final Log LOG = LogFactory.getLog(NativeIO.class);
private static boolean nativeLoaded = false;
private static boolean fadvisePossible = true;
private static boolean syncFileRangePossible = true;
static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
"hadoop.workaround.non.threadsafe.getpwuid";
static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = true;
private static long cacheTimeout = -1;
private static CacheManipulator cacheManipulator = new CacheManipulator();
public static CacheManipulator getCacheManipulator() {
return cacheManipulator;
}
public static void setCacheManipulator(CacheManipulator cacheManipulator) {
POSIX.cacheManipulator = cacheManipulator;
}
/**
* Used to manipulate the operating system cache.
*/
@VisibleForTesting
public static class CacheManipulator {
public void mlock(String identifier, ByteBuffer buffer,
long len) throws IOException {
POSIX.mlock(buffer, len);
}
public long getMemlockLimit() {
return NativeIO.getMemlockLimit();
}
public long getOperatingSystemPageSize() {
return NativeIO.getOperatingSystemPageSize();
}
public void posixFadviseIfPossible(String identifier,
FileDescriptor fd, long offset, long len, int flags)
throws NativeIOException {
NativeIO.POSIX.posixFadviseIfPossible(identifier, fd, offset,
len, flags);
}
public boolean verifyCanMlock() {
return NativeIO.isAvailable();
}
}
/**
* A CacheManipulator used for testing which does not actually call mlock.
* This allows many tests to be run even when the operating system does not
* allow mlock, or only allows limited mlocking.
*/
@VisibleForTesting
public static class NoMlockCacheManipulator extends CacheManipulator {
public void mlock(String identifier, ByteBuffer buffer,
long len) throws IOException {
LOG.info("mlocking " + identifier);
}
public long getMemlockLimit() {
return 1125899906842624L;
}
public long getOperatingSystemPageSize() {
return 4096;
}
public boolean verifyCanMlock() {
return true;
}
}
static {
if (NativeCodeLoader.isNativeCodeLoaded()) {
try {
Configuration conf = new Configuration();
workaroundNonThreadSafePasswdCalls = conf.getBoolean(
WORKAROUND_NON_THREADSAFE_CALLS_KEY,
WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);
initNative();
nativeLoaded = true;
cacheTimeout = conf.getLong(
CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,
CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) *
1000;
LOG.debug("Initialized cache for IDs to User/Group mapping with a " +
" cache timeout of " + cacheTimeout/1000 + " seconds.");
} catch (Throwable t) {
没有合适的资源?快使用搜索试试~ 我知道了~
资源推荐
资源详情
资源评论
收起资源包目录
HadoopMRunit_Winutils (104个子文件)
.gitignore 77B
HadoopMRunit_Winutils_20150520.iml 9KB
NativeIO.java 37KB
WordCountApp.java 3KB
WordCountAppTest.java 2KB
.name 30B
log4j.properties 1KB
workspace.xml 51KB
uiDesigner.xml 9KB
pom.xml 3KB
yarn-site.xml 2KB
hdfs-site.xml 2KB
mapred-site.xml 2KB
misc.xml 1KB
compiler.xml 1KB
Maven__org_apache_hadoop_hadoop_mapreduce_client_jobclient_2_6_0.xml 706B
Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml 704B
Maven__org_apache_hadoop_hadoop_mapreduce_client_shuffle_2_6_0.xml 692B
Maven__org_apache_hadoop_hadoop_mapreduce_client_common_2_6_0.xml 685B
Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_6_0.xml 671B
Maven__org_apache_hadoop_hadoop_mapreduce_client_app_2_6_0.xml 664B
Maven__org_apache_hadoop_hadoop_yarn_server_common_2_6_0.xml 650B
Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml 634B
Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml 629B
Maven__commons_configuration_commons_configuration_1_6.xml 624B
Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml 620B
Maven__commons_collections_commons_collections_3_2_1.xml 616B
Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml 615B
Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml 606B
Maven__org_apache_hadoop_hadoop_yarn_client_2_6_0.xml 601B
Maven__org_apache_hadoop_hadoop_annotations_2_6_0.xml 601B
Maven__org_apache_hadoop_hadoop_yarn_common_2_6_0.xml 601B
Maven__org_apache_curator_curator_framework_2_6_0.xml 598B
Maven__commons_beanutils_commons_beanutils_1_7_0.xml 594B
Maven__org_fusesource_leveldbjni_leveldbjni_all_1_8.xml 591B
Maven__org_apache_commons_commons_compress_1_4_1.xml 591B
Maven__commons_httpclient_commons_httpclient_3_1.xml 591B
Maven__org_apache_directory_api_api_util_1_0_0_M20.xml 587B
Maven__org_apache_curator_curator_recipes_2_6_0.xml 584B
Maven__org_apache_hadoop_hadoop_yarn_api_2_6_0.xml 580B
Maven__org_codehaus_jackson_jackson_jaxrs_1_8_3.xml 578B
Maven__com_jamesmurty_utils_java_xmlbuilder_0_4.xml 578B
Maven__org_apache_httpcomponents_httpclient_4_1_2.xml 577B
Maven__org_apache_curator_curator_client_2_6_0.xml 577B
Maven__com_google_protobuf_protobuf_java_2_5_0.xml 574B
Maven__commons_logging_commons_logging_1_1_3.xml 572B
Maven__org_apache_commons_commons_math3_3_1_1.xml 570B
Maven__commons_digester_commons_digester_1_8.xml 569B
Maven__commons_daemon_commons_daemon_1_0_13.xml 568B
Maven__org_apache_hadoop_hadoop_common_2_6_0.xml 566B
Maven__org_xerial_snappy_snappy_java_1_0_4_1.xml 566B
Maven__org_apache_hadoop_hadoop_client_2_6_0.xml 566B
Maven__org_apache_httpcomponents_httpcore_4_1_2.xml 563B
Maven__com_thoughtworks_paranamer_paranamer_2_3.xml 560B
Maven__org_codehaus_jackson_jackson_xc_1_8_3.xml 557B
Maven__org_mortbay_jetty_jetty_util_6_1_26.xml 552B
Maven__org_apache_hadoop_hadoop_auth_2_6_0.xml 552B
Maven__org_apache_hadoop_hadoop_hdfs_2_6_0.xml 552B
Maven__org_apache_zookeeper_zookeeper_3_4_6.xml 550B
Maven__com_sun_xml_bind_jaxb_impl_2_2_3_1.xml 548B
Maven__com_google_code_findbugs_jsr305_1_3_9.xml 545B
Maven__tomcat_jasper_compiler_5_5_23.xml 543B
Maven__com_sun_jersey_jersey_client_1_9.xml 540B
Maven__com_sun_jersey_jersey_server_1_9.xml 540B
Maven__commons_codec_commons_codec_1_4.xml 536B
Maven__tomcat_jasper_runtime_5_5_23.xml 536B
Maven__org_slf4j_slf4j_log4j12_1_7_5.xml 534B
Maven__org_codehaus_jettison_jettison_1_1.xml 533B
Maven__org_apache_mrunit_mrunit_hadoop2_1_0_0.xml 533B
Maven__org_hamcrest_hamcrest_core_1_3.xml 532B
Maven__org_mockito_mockito_all_1_8_5.xml 528B
Maven__javax_xml_stream_stax_api_1_0_2.xml 527B
Maven__javax_activation_activation_1_1.xml 527B
Maven__com_sun_jersey_jersey_json_1_9.xml 526B
Maven__com_sun_jersey_jersey_core_1_9.xml 526B
Maven__net_java_dev_jets3t_jets3t_0_9_0.xml 525B
Maven__commons_lang_commons_lang_2_6.xml 525B
Maven__org_htrace_htrace_core_3_0_4.xml 524B
Maven__javax_servlet_servlet_api_2_5.xml 522B
Maven__javax_xml_bind_jaxb_api_2_2_2.xml 519B
Maven__org_mortbay_jetty_jetty_6_1_26.xml 517B
Maven__io_netty_netty_3_6_2_Final.xml 516B
Maven__com_google_code_gson_gson_2_2_4.xml 515B
Maven__commons_net_commons_net_3_1.xml 514B
Maven__commons_cli_commons_cli_1_2.xml 514B
Maven__com_google_guava_guava_11_0_2.xml 513B
Maven__javax_servlet_jsp_jsp_api_2_1.xml 510B
Maven__org_slf4j_slf4j_api_1_7_5.xml 506B
Maven__commons_el_commons_el_1_0.xml 503B
Maven__commons_io_commons_io_2_4.xml 503B
Maven__xml_apis_xml_apis_1_3_04.xml 502B
Maven__xerces_xercesImpl_2_9_1.xml 501B
Maven__org_apache_avro_avro_1_7_4.xml 495B
Maven__com_jcraft_jsch_0_1_42.xml 482B
Maven__log4j_log4j_1_2_17.xml 469B
core-site.xml 467B
Maven__xmlenc_xmlenc_0_52.xml 466B
Maven__junit_junit_4_12.xml 455B
Maven__org_tukaani_xz_1_0.xml 451B
Maven__asm_asm_3_1.xml 426B
共 104 条
- 1
- 2
资源评论
xiaoxiaomo_com
- 粉丝: 7
- 资源: 34
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
最新资源
- 课程设计-python爬虫-爬取日报,爬取日报文章后存储到本地,附带源代码+课程设计报告
- 软件和信息技术服务行业投资与前景预测.pptx
- 课程设计-基于SpringBoot + Mybatis+python爬虫NBA球员数据爬取可视化+源代码+文档+sql+效果图
- 软件品质管理系列二项目策划规范.doc
- 基于TensorFlow+PyQt+GUI的酒店评论情感分析,支持分析本地数据文件和网络爬取数据分析+源代码+文档说明+安装教程
- 软件定义无线电中的模拟电路测试技术.pptx
- 软件开发协议(作为技术开发合同附件).doc
- 软件开发和咨询行业技术趋势分析.pptx
- 软件测试题详解及答案.doc
- 软件漏洞生命周期管理策略.pptx
资源上传下载、课程学习等过程中有任何疑问或建议,欢迎提出宝贵意见哦~我们会及时处理!
点击此处反馈
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功