AI导航网
微信公众号搜"智元新知"关注
微信扫一扫可直接关注哦！

hadoop学习之路(4)

分类：Hadoop作者：编程之家原文地址

1.hdfs的文件操作

1.1创建maven项目

@H_404_4@

@H_404_4@

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.zs</groupId>
    <artifactId>HDFS</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.2</version>
        </dependency>
<!--        <dependency>-->
<!--            <groupId>jdk.tools</groupId>-->
<!--            <artifactId>jdk.tools</artifactId>-->
<!--            <version>1.8</version>-->
<!--            <scope>system</scope>-->
<!--            <systemPath>D:/install/Java/jdk1.8.0_281/lib/tools.jar</systemPath>-->
<!--        </dependency>-->

    </dependencies>


</project>
View Code

1.2配置core-site.xml

@H_404_4@

@H_404_4@

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <!-- 指定HDFS中NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop101:9000</value>
    </property>

    <!-- 指定Hadoop运行时产生文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-2.7.2/data/tmp</value>
    </property>
    <!--    客戶端使用-->
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
</configuration>
View Code

1.3添加log4j配置文件

@H_404_4@

@H_404_4@

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
View Code

1.4添加hdfs测试类

@H_404_4@

@H_404_4@

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @Author: layman
 * @Date:Create：in 2021/3/7 10:34
 * @Description:
 */
public class TestHDFS {
    private FileSystem fs;
    Configuration conf = new Configuration();

    @Before
    public void init() throws IOException, URISyntaxException {
        fs = FileSystem.get(new URI("hdfs://hadoop101:9000"), conf);
    }

    @After
    public void close() throws IOException {
        if (fs != null) {
            fs.close();
        }
    }

    @Test
    public void testMkdir() throws IOException, InterruptedException, URISyntaxException {

        System.out.println(fs.getClass().getName());
        fs.mkdirs(new Path("/idea1"));
        fs.close();
    }

    //    上传文件
    @Test
    public void testUpload() throws IOException {
        fs.copyFromLocalFile(false, true, new Path("D:\\idea_workplace\\hadoop\\HDFS\\jdk-8u241-linux-x64.tar.gz"), new Path("/"));
    }

    @Test
    public void testDownload() throws IOException {
        fs.copyToLocalFile(false, new Path("/wcinput"), new Path("D:/idea_workplace/hadoop/HDFS"), true);
    }

    @Test
    public void testDelete() throws IOException {
        fs.delete(new Path("/idea1"), true);
    }

    @Test
    public void testRename() throws IOException {
        fs.rename(new Path("/idea"), new Path("/ideaDir"));
    }

    @Test
    public void testIfPathExits() throws IOException {
        System.out.println(fs.exists(new Path("/ideaDir")));
        System.out.println(fs.exists(new Path("/idea")));
    }

    @Test
    public void testDirorFile() throws IOException {
        Path path = new Path("/ideaDir");
//        System.out.println(fs.isDirectory(path));
//        System.out.println(fs.isFile(path));
        FileStatus fileStatus = fs.getFileStatus(path);
        FileStatus[] fileStatuses = fs.listStatus(new Path("/wcoutput"));
        for (FileStatus status : fileStatuses) {
            Path path1 = status.getPath();
            System.out.println(path1 + "目录:" + fileStatus.isDirectory());
            System.out.println(path1 + "文件:" + fileStatus.isFile());
        }
//        System.out.println("目录:"+fileStatus.isDirectory());
//        System.out.println("文件:"+fileStatus.isFile());
    }

    @Test
    public void testGetBlock() throws IOException {
        Path path = new Path("/jdk-8u241-linux-x64.tar.gz");
        RemoteIterator<LocatedFileStatus> status = fs.listLocatedStatus(path);
        while (status.hasNext()) {
            LocatedFileStatus locatedFileStatus = status.next();
            System.out.println(locatedFileStatus);
            BlockLocation[] blockLocations = locatedFileStatus.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation);
            }

        }

    }

}
View Code

@H_404_4@

 

 

 为了区分bash和java代码,将用户名设置为本机和layman,设置执行权限

2.MR

1-3不变

2.4编写MR程序

2.4.1 

@H_404_4@

@H_404_4@

package com.zs.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @Author: layman
 * @Date:Create：in 2021/3/7 16:38
 * @Description:
 */
public class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    private Text out_key = new Text();
    private IntWritable out_value = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        System.out.println("keyin:" + key + "keyout:" + value);
        String[] words = value.toString().split("\t");
        for (String word : words) {
            out_key.set(word);
            context.write(out_key, out_value);
        }
    }
}
View Code @H_404_4@

@H_404_4@

package com.zs.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * @Author: layman
 * @Date:Create：in 2021/3/7 16:56
 * @Description:
 */
public class WCReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    private IntWritable out_value = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for (IntWritable value : values) {
            sum += value.get();
        }
        out_value.set(sum);
        context.write(key, out_value);
    }
}
View Code @H_404_4@

@H_404_4@

package com.zs.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @Author: layman
 * @Date:Create：in 2021/3/7 17:03
 * @Description:
 */
public class WCDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException {
//        Path inputPath = new Path("D:/idea_workplace/hadoop/MapReduce/mrinput");
//        Path outputPath = new Path("D:/idea_workplace/hadoop/MapReduce/mroutput");
        Path inputPath = new Path("/input/mrinput");
        Path outputPath = new Path("/output/mroutput/wordcount");
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://hadoop101:9000");
        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(outputPath)) {
            fs.delete(outputPath, true);
        }
        Job job = Job.getInstance(conf);
        job.setJobName("wordcout");
        job.setMapperClass(WCMapper.class);
        job.setReducerClass(WCReducer.class);
        job.setoutputKeyClass(Text.class);
        job.setoutputValueClass(IntWritable.class);
        FileInputFormat.setInputPaths(job, inputPath);
        FileOutputFormat.setoutputPath(job, outputPath);
        job.waitForCompletion(true);
    }
}
View Code @H_404_4@

 

 设置文件测试utf-8,不选utf-8-BOM,否则是按行的效果计算

@H_404_4@

 

版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 [email protected] 举报，一经查实，本站将立刻删除。

相关推荐

hadoop day2-搭建
hadoop搭建准备工作三台虚拟机：master、node1、node2检查时间是否同步：date检查java的jdk是否被安装好：java-version修改主机名三台分别执行vim/etc/hostname并将内容指定为对应的主机名 关闭防火墙：systemctlst...
Hadoop 二十三
文件的更名和移动:    获取文件详细信息       遇到的问题：不能直接在web上上传文件。   权限问题：修改后即可正常创建  参考：https://blog.csdn.net/weixin_44575660/article/details/118687993
大数据Hadoop之——HDFS小文件问题与处理实战操作
目录一、背景1）小文件是如何产生的？2）文件块大小设置3）HDFS分块目的二、HDFS小文件问题处理方案1）HadoopArchive（HAR）2）Sequencefile3）CombineFileInputFormat4）开启JVM重用5）合并本地的小文件，上传到HDF...
大数据Hadoop之——Hadoop HDFS多目录磁盘扩展与数据平衡实战操作
目录一、概述二、HadoopDataNode多目录磁盘配置1）配置hdfs-site.xml2）配置详解1、dfs.datanode.data.dir2、dfs.datanode.fsdataset.volume.choosing.policy3、dfs.datanode.available-space-volume-choosing-polic...
平台搭建伪分布式
平台搭建（伪分布式）伪分布式搭建在VM中搭建std-master修改配置文件centos7-cl1.vmdkstd-master.vmx-将配置文件中vm的版本号改成自己电脑对应的vm版本修改客户端的操作系统为centos764位打开虚拟机修改虚拟机网络cd...
Harley浅谈HadoopHDFS
 一、HDFS概述 1.1、HDFS产出背景及定义 1.1.1、HDFS产生背景   随着数据量越来越大，在一个操作系统存不下所有的数据，那么就分配到更多的操作系统管理的磁盘中，但是不方便管理和维护，迫切需要一种系统来管...
Hadoop 十二
配置workers进入hadoop/etc/hadoop  编辑workers文件  然后分发给另外两个服务器     准备启动集群第一次需要初始化.  初始化完成后增加了data文件，  进入上面那个路径，就能看到当前服务器的版本号 ...
第六周总结8.13
这周我对ssm框架进行了更深一步的开发，加入了多用户，并对除登录外的请求进行了拦截，这样用户在未登录的时候是访问不到资源的。并且对hadoop进行了初步的学习，包括虚拟机的安装等等。下周会对hadoop进行更深一步的...
安装Hadoop2.10.1
前言通过在Hadoop1安装Hadoop,然后配置相应的配置文件，最后将Hadoop所有文件同步到其他Hadoop节点。一、集群规划#主机名‘master/hadoop1’‘slave01/hadoop2’‘slave02/hadoop3’#启动节点NamenodeNodemanagerNod...
集群崩溃处理
1.先杀死进程（先进入到hadoop版本文件里，我的是/opt/module/hadoop-3.1.3/）sbin/stop-dfs.sh2.删除每个集群上的data以及logsrm-rfdata/logs/3.格式化hdfsnamenode-format4.再启动sbin/sart-dfs.sh
    pythonJavaScriptjavaHTMLreactjsC#AndroidCSSNode.jssqlrpython-3.xMysqLjQueryc++pandasFlutterangularIOSdjangolinuxswifttypescript路由器JSON路由器设置无线路由器h3c华三华三路由器设置华三路由器电脑软件教程arraysdocker软件图文教程Cvue.jslaravelspring-bootreact-native