Pages

Sunday, September 20, 2015

Fast I/O with Java : Good for online programming contest

online programming contest

While we are doing coding in online any of the platform (codechef , Hackerrank , Topcoder ), there is always a need to do faster input output because here we are dealing with top coder of the world. Also there is need to minimize the time for input/output for better time complexity.

So, As a newcomer I have also faced the similar problem for slower input/output for this I am posting the optimized Java Code that people can use with there code.

package com.io;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.InputMismatchException;

public class IO {

 public static void main(String[] args) {
  
  InputReader input = new InputReader(System.in);
  int num = input.readInt();
  
  OutputWriter output = new OutputWriter(System.out);
  output.print(num);
  output.flush();
  output.close();
 }

    private static class InputReader
    {
        private InputStream stream;
        private byte[] buf = new byte[1024];
        private int curChar;
        private int numChars;
        private SpaceCharFilter filter;

        public InputReader(InputStream stream)
        {
            this.stream = stream;
        }

        public int read()
        {
            if (numChars == -1)
                throw new InputMismatchException();
            if (curChar >= numChars)
            {
                curChar = 0;
                try
                {
                    numChars = stream.read(buf);
                } catch (IOException e)
                {
                    throw new InputMismatchException();
                }
                if (numChars <= 0)
                    return -1;
            }
            return buf[curChar++];
        }

        public int readInt()
        {
            int c = read();
            while (isSpaceChar(c))
                c = read();
            int sgn = 1;
            if (c == '-')
            {
                sgn = -1;
                c = read();
            }
            int res = 0;
            do
            {
                if (c < '0' || c > '9')
                    throw new InputMismatchException();
                res *= 10;
                res += c - '0';
                c = read();
            } while (!isSpaceChar(c));
            return res * sgn;
        }

        public String readString()
        {
            int c = read();
            while (isSpaceChar(c))
                c = read();
            StringBuilder res = new StringBuilder();
            do
            {
                res.appendCodePoint(c);
                c = read();
            } while (!isSpaceChar(c));
            return res.toString();
        }
        public double readDouble() {
            int c = read();
            while (isSpaceChar(c))
                c = read();
            int sgn = 1;
            if (c == '-') {
                sgn = -1;
                c = read();
            }
            double res = 0;
            while (!isSpaceChar(c) && c != '.') {
                if (c == 'e' || c == 'E')
                    return res * Math.pow(10, readInt());
                if (c < '0' || c > '9')
                    throw new InputMismatchException();
                res *= 10;
                res += c - '0';
                c = read();
            }
            if (c == '.') {
                c = read();
                double m = 1;
                while (!isSpaceChar(c)) {
                    if (c == 'e' || c == 'E')
                        return res * Math.pow(10, readInt());
                    if (c < '0' || c > '9')
                        throw new InputMismatchException();
                    m /= 10;
                    res += (c - '0') * m;
                    c = read();
                }
            }
            return res * sgn;
        }
        public long readLong() {
            int c = read();
            while (isSpaceChar(c))
                c = read();
            int sgn = 1;
            if (c == '-') {
                sgn = -1;
                c = read();
            }
            long res = 0;
            do {
                if (c < '0' || c > '9')
                    throw new InputMismatchException();
                res *= 10;
                res += c - '0';
                c = read();
            } while (!isSpaceChar(c));
            return res * sgn;
        }
        public boolean isSpaceChar(int c)
        {
            if (filter != null)
                return filter.isSpaceChar(c);
            return c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == -1;
        }

        public String next()
        {
            return readString();
        }

        public interface SpaceCharFilter
        {
            public boolean isSpaceChar(int ch);
        }
    }

    private static class OutputWriter
    {
        private final PrintWriter writer;

        public OutputWriter(OutputStream outputStream)
        {
            writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(outputStream)));
        }

        public OutputWriter(Writer writer)
        {
            this.writer = new PrintWriter(writer);
        }

        public void print(Object... objects)
        {
            for (int i = 0; i < objects.length; i++)
            {
                if (i != 0)
                    writer.print(' ');
                writer.print(objects[i]);
            }
        }

        public void printLine(Object... objects)
        {
            print(objects);
            writer.println();
        }

        public void close()
        {
            writer.close();
        }

        public void flush()
        {
            writer.flush();
        }

    }
}


you need to put the static classes InputReader and OutputWriter with there existing code and use these classes with their objects in the main method as shown in the above code and you will find that the time taken for input/output has been increases surprisingly.

Please, leave your comment if you are facing any difficulty while using the above code.


Saturday, August 15, 2015

How to run php map reduce job using hadoop streaming

Hadoop Streaming


Hadoop streaming is a feature that comes with the Apache Hadoop distribution. The feature allows you to create and run Map/Reduce jobs with any executable or script as the mapper and/or the reducer.

How Hadoop Streaming works ?


For Mapper : When an executable is specified for mappers, each mapper task will launch the executable as a separate process when the mapper is initialized. As the mapper task runs, it converts its inputs into lines and feed the lines to the stdin of the process. In the meantime, the mapper collects the line oriented outputs from the stdout of the process and converts each line into a key/value pair, which is collected as the output of the mapper. By default, the prefix of a line up to the first tab character is the key and the rest of the line (excluding the tab character) will be the value. If there is no tab character in the line, then entire line is considered as key and the value is null.

For Reducer :When an executable is specified for reducers, each reducer task will launch the executable as a separate process then the reducer is initialized. As the reducer task runs, it converts its input key/values pairs into lines and feeds the lines to the stdin of the process. In the meantime, the reducer collects the line oriented outputs from the stdout of the process, converts each line into a key/value pair, which is collected as the output of the reducer. By default, the prefix of a line up to the first tab character is the key and the rest of the line is the value.

Hadoop Streaming is going to be very helpful to those people who has limited knowledge over java,d
because they just need to write two scripts i.e. one for the mapper and reducer.

For this the script that I going to use php scripting language for the demonstrative purpose.

So people who knows basics of the php are able to run map-reduce job with the help of the hadoop streaming.

Before we start I am assuming that php compiler is installed on your system. you can download the php compiler from here.

NOTE : php compiler should be installed in any of the hadoop cluster node

If you don't know how to setup apache  hadoop on your system you can take the help of my previous blog on How to Setup Apache Hadoop

Now lets start, how to start with your first map task, I am assuming that $HADOOP_HOME is setup in your system, so to run the mapper task you have enter the following command :

$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-streaming-1.2.1.jar 
-input /input 
-output /output 
-mapper "php map.php" 
-file map.php

while we are going to run our first mapper we have provide certain parameters these are :

-input -> Path to input folder with respect to hadoop file system
-output -> Path to output folder with respect to hadoop file system
-mapper -> command you need to run for script specific file, for e.g. php file can be run on the command line by the command "php myfile.php"
-file -> Path to php script in the local file system

Input file should be in a format so that it can be easily understand by hadoop.

suppose we want to count number of times word hadoop in a text file , that text file can be taken as input, we can put it as follows :

Apache hadoop Apache hadoop Apache hadoop Apache hadoop Apache hadoop

We have to convert this file into key-value format, we can split this by space and and each word is assigned a count 1, to make it like this :

Apache    1
hadoop    1
Apache    1
hadoop    1
Apache    1
hadoop    1
Apache    1
hadoop    1

In the myfile.php, we have to write some code so that hadoop can understand it.

<?php
while (!feof(STDIN)) {

$line = fgets(STDIN);
$split_line = $line.split(" ");
    foreach($split_line as $key){
    echo $split_line."\t"."1";
    }
}


the above text read the input from the input stream using STDIN and passing it to the output stream using echo,with this all the text content is converted to the key-value format.

Saturday, August 8, 2015

Adding Hadoop Slave Node to Running Hadoop Cluster



Hello readers today I am going to show you how you can add hadoop slave node to the running hadoop cluster, in the previous tutorial on Crawling with Apache Nutch and Apache Hadoop , where we have setup the apache nutch crawler over apache hadoop cluster.

If we are using any crawler that crawls over the web and day by day data will increase exponentially,

Due to limited number of the computational resources (slave nodes) in the hadoop cluster, the processing of the data take more time. To speed up the process it is better to add one or more slave node, this will increase the processing power to our data processing but also prevent blocking of the CPU threads.

Suppose your hadoop cluster is running with 1 datanode and 1 tasktracker and you want to add one more data node to this cluster.

Follow the steps to configure the new data node :

1.  configure slave node with the help of this tutorial, this is similar to configuring the master node.

2.  login to master node using ssh

ssh huser@master

3.  Add IP address of the slave node to the master slave text file with the help of text editor (here i am using vim)

huser@master$ vim $HADOOP_HOME/conf/slave

4.  copy all the configuration from the master node to new slave node using scp shell command including ssh public key

huser@master$ scp /home/huser/.ssh/authorized_keys $HADOOP_HOME/conf/* huser@slave:/home/huser/$HADOOP_HOME/conf/

5.  start the datanode

huser@slave$ $HADOOP_HOME/bin/hadoop start datanode

6.  start the tasktracker

huser@slave$ $HADOOP_HOME/bin/hadoop start tasktracker

After this you new data node will automatically discovered by the namenode and the new map/reduce task will assign to it if any job is running.

When the new data node is added, then it can have only new blocks but to make proper use of new data node the total number of the block to be equally distributed, this can be done with the help of the balancer.

Following command can be used to balance the total blocks in the cluster

huser@slave$ $HADOOP_HOME/bin/start-balancer.sh