OOM 与内存问题排查

`Object obj = new Object()
`

`static Map<User,Long> userVisitMap = new HashMap<>();
...
userVisitMap.put(user, time);
`

`// 伪代码
Object object = new Object();
SoftReference<Object> softRef = new SoftReference(object);
`

`-XX:SoftRefLRUPolicyMSPerMB=<N>
`

`// 伪代码
Object object = new Object();
WeakReference<Object> softRef = new WeakReference(object);
`

`Object  object = new Object();
ReferenceQueue queue = new ReferenceQueue();
// 虚引用，必须与一个引用队列关联
PhantomReference pr = new PhantomReference(object, queue);
`

`private static void startMonitoring(ReferenceQueue<MyObject> referenceQueue, Reference<MyObject> ref) {
     ExecutorService ex = Executors.newSingleThreadExecutor();
     ex.execute(() -> {
         while (referenceQueue.poll()!=ref) {
             //don't hang forever
             if(finishFlag){
                 break;
            }
        }
         System.out.println("-- ref gc'ed --");

    });
     ex.shutdown();
}
`

import com.sun.net.httpserver.HttpContext;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.OutputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
public class OOMTest {
   public static final int _1MB = 1024 * 1024;
   static List<byte[]> byteList = new ArrayList<>();
   private static void oom(HttpExchange exchange) {
       try {
           String response = "oom begin!";
           exchange.sendResponseHeaders(200, response.getBytes().length);
           OutputStream os = exchange.getResponseBody();
           os.write(response.getBytes());
           os.close();
       } catch (Exception ex) {
       }
       for (int i = 0; ; i++) {
           byte[] bytes = new byte[_1MB];
           byteList.add(bytes);
           System.out.println(i + "MB");
           memPrint();
           try {
               Thread.sleep(1000);
           } catch (Exception e) {
           }
       }
   }
   static void memPrint() {
       for (MemoryPoolMXBean memoryPoolMXBean : ManagementFactory.getMemoryPoolMXBeans()) {
           System.out.println(memoryPoolMXBean.getName() +
                   "  committed:" + memoryPoolMXBean.getUsage().getCommitted() +
                   "  used:" + memoryPoolMXBean.getUsage().getUsed());
       }
   }
   private static void srv() throws Exception {
       HttpServer server = HttpServer.create(new InetSocketAddress(8888), 0);
       HttpContext context = server.createContext("/");
       context.setHandler(OOMTest::oom);
       server.start();
   }
   public static void main(String[] args) throws Exception{
       srv();
   }
}

import com.sun.net.httpserver.HttpContext;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.OutputStream;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.HashMap;
import java.util.Map;
public class MetaspaceOOMTest {
   public interface Facade {
       void m(String input);
   }
   public static class FacadeImpl implements Facade {
       @Override
       public void m(String name) {
       }
   }
   public static class MetaspaceFacadeInvocationHandler implements InvocationHandler {
       private Object impl;
       public MetaspaceFacadeInvocationHandler(Object impl) {
           this.impl = impl;
       }
       @Override
       public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
           return method.invoke(impl, args);
       }
   }
   private static Map<String, Facade> classLeakingMap = new HashMap<String, Facade>();
   private static void oom(HttpExchange exchange) {
       try {
           String response = "oom begin!";
           exchange.sendResponseHeaders(200, response.getBytes().length);
           OutputStream os = exchange.getResponseBody();
           os.write(response.getBytes());
           os.close();
       } catch (Exception ex) {
       }
       try {
           for (int i = 0; ; i++) {
               String jar = "file:" + i + ".jar";
               URL[] urls = new URL[]{new URL(jar)};
               URLClassLoader newClassLoader = new URLClassLoader(urls);
               Facade t = (Facade) Proxy.newProxyInstance(newClassLoader,
                       new Class<?>[]{Facade.class},
                       new MetaspaceFacadeInvocationHandler(new FacadeImpl()));
               classLeakingMap.put(jar, t);
           }
       } catch (Exception e) {
       }
   }
   private static void srv() throws Exception {
       HttpServer server = HttpServer.create(new InetSocketAddress(8888), 0);
       HttpContext context = server.createContext("/");
       context.setHandler(MetaspaceOOMTest::oom);
       server.start();
   }
   public static void main(String[] args) throws Exception {
       srv();
   }
}

import com.sun.net.httpserver.HttpContext;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.OutputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
public class OffHeapOOMTest {
   public static final int _1MB = 1024 * 1024;
   static List<ByteBuffer> byteList = new ArrayList<>();
   private static void oom(HttpExchange exchange) {
       try {
           String response = "oom begin!";
           exchange.sendResponseHeaders(200, response.getBytes().length);
           OutputStream os = exchange.getResponseBody();
           os.write(response.getBytes());
           os.close();
       } catch (Exception ex) {
       }
       for (int i = 0; ; i++) {
           ByteBuffer buffer = ByteBuffer.allocateDirect(_1MB);
           byteList.add(buffer);
           System.out.println(i + "MB");
           memPrint();
           try {
               Thread.sleep(1000);
           } catch (Exception e) {
           }
       }
   }
   private static void srv() throws Exception {
       HttpServer server = HttpServer.create(new InetSocketAddress(8888), 0);
       HttpContext context = server.createContext("/");
       context.setHandler(OffHeapOOMTest::oom);
       server.start();
   }
   public static void main(String[] args) throws Exception {
       srv();
   }
   static void memPrint() {
       for (MemoryPoolMXBean memoryPoolMXBean : ManagementFactory.getMemoryPoolMXBeans()) {
           System.out.println(memoryPoolMXBean.getName() +
                   "  committed:" + memoryPoolMXBean.getUsage().getCommitted() +
                   "  used:" + memoryPoolMXBean.getUsage().getUsed());
       }
   }
}

public class StackOverflowTest {
   static int count = 0;
   static void a() {
       System.out.println(count);
       count++;
       b();
   }
   static void b() {
       System.out.println(count);
       count++;
       a();
   }
   public static void main(String[] args) throws Exception {
       a();
   }
}

top

top -Hp $pid

printf %x $tid

jstack $pid >$pid.log

less $pid.log

ss -antp > $DUMP_DIR/ss.dump 2>&1

netstat -s > $DUMP_DIR/netstat-s.dump 2>&1

sar -n DEV 1 2 > $DUMP_DIR/sar-traffic.dump 2>&1

lsof -p $PID > $DUMP_DIR/lsof-$PID.dump

mpstat > $DUMP_DIR/mpstat.dump 2>&1
vmstat 1 3 > $DUMP_DIR/vmstat.dump 2>&1
sar -p ALL  > $DUMP_DIR/sar-cpu.dump  2>&1
uptime > $DUMP_DIR/uptime.dump 2>&1

iostat -x > $DUMP_DIR/iostat.dump 2>&1

free -h > $DUMP_DIR/free.dump 2>&1

ps -ef > $DUMP_DIR/ps.dump 2>&1
dmesg > $DUMP_DIR/dmesg.dump 2>&1
sysctl -a > $DUMP_DIR/sysctl.dump 2>&1

${JDK_BIN}jinfo $PID > $DUMP_DIR/jinfo.dump 2>&1

${JDK_BIN}jstat -gcutil $PID > $DUMP_DIR/jstat-gcutil.dump 2>&1
${JDK_BIN}jstat -gccapacity $PID > $DUMP_DIR/jstat-gccapacity.dump 2>&1

${JDK_BIN}jmap $PID > $DUMP_DIR/jmap.dump 2>&1
${JDK_BIN}jmap -heap $PID > $DUMP_DIR/jmap-heap.dump 2>&1
${JDK_BIN}jmap -histo $PID > $DUMP_DIR/jmap-histo.dump 2>&1
${JDK_BIN}jmap -dump:format=b,file=$DUMP_DIR/heap.bin $PID > /dev/null  2>&1

${JDK_BIN}jstack $PID > $DUMP_DIR/jstack.dump 2>&1

top -Hp $PID -b -n 1 -c >  $DUMP_DIR/top-$PID.dump 2>&1

kill -3 $PID

gcore -o $DUMP_DIR/core $PID

${JDK_BIN}jhsdb jmap --exe ${JDK}java  --core $DUMP_DIR/core --binaryheap

jhsdb jmap  --heap --pid  37340
jhsdb jmap  --pid  37288
jhsdb jmap  --histo --pid  37340
jhsdb jmap  --binaryheap --pid  37340

find / | grep "x"

//leak example
import java.util.HashMap;
import java.util.Map;

public class HashMapLeakDemo {
    public static class Key {
        String title;

public Key(String title) {
            this.title = title;
        }
    }

public static void main(String[] args) {
        Map<Key, Integer> map = new HashMap<>();

map.put(new Key("1"), 1);
        map.put(new Key("2"), 2);
        map.put(new Key("3"), 2);

        Integer integer = map.get(new Key("2"));
        System.out.println(integer);
    }
}

import java.util.UUID;

public class InternDemo {
    static String getLongStr() {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < 100000; i++) {
            sb.append(UUID.randomUUID().toString());
        }
        return sb.toString();
    }

    public static void main(String[] args) {
        while (true) {
            getLongStr().intern();
        }
    }
}

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;

public class Objects4MAT {

static class A4MAT {
        B4MAT b4MAT = new B4MAT();
    }

static class B4MAT {
        C4MAT c4MAT = new C4MAT();
    }

static class C4MAT {
        List<String> list = new ArrayList<>();
    }

static class DominatorTreeDemo1 {
        DominatorTreeDemo2 dominatorTreeDemo2;

public void setValue(DominatorTreeDemo2 value) {
            this.dominatorTreeDemo2 = value;
        }
    }

static class DominatorTreeDemo2 {
        DominatorTreeDemo1 dominatorTreeDemo1;

public void setValue(DominatorTreeDemo1 value) {
            this.dominatorTreeDemo1 = value;
        }
    }

static class Holder {
        DominatorTreeDemo1 demo1 = new DominatorTreeDemo1();
        DominatorTreeDemo2 demo2 = new DominatorTreeDemo2();

Holder() {
            demo1.setValue(demo2);
            demo2.setValue(demo1);
        }

private boolean aBoolean = false;
        private char aChar = '\0';
        private short aShort = 1;
        private int anInt = 1;
        private long aLong = 1L;
        private float aFloat = 1.0F;
        private double aDouble = 1.0D;
        private Double aDouble_2 = 1.0D;
        private int[] ints = new int[2];
        private String string = "1234";
    }

Runnable runnable = () -> {
        Map<String, A4MAT> map = new HashMap<>();

IntStream.range(0, 100).forEach(i -> {
            byte[] bytes = new byte[1024 * 1024];
            String str = new String(bytes).replace('\0', (char) i);
            A4MAT a4MAT = new A4MAT();
            a4MAT.b4MAT.c4MAT.list.add(str);

map.put(i + "", a4MAT);
        });

Holder holder = new Holder();

try {
            //sleep forever , retain the memory
            Thread.sleep(Integer.MAX_VALUE);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    };

void startHugeThread() throws Exception {
        new Thread(runnable, "huge-thread").start();
    }

    public static void main(String[] args) throws Exception {
        Objects4MAT objects4MAT = new Objects4MAT();
        objects4MAT.startHugeThread();
    }
}

public class DorminatorTreeDemo {
    static class A {
        C c;

byte[] data = new byte[1024 * 1024 * 2];
    }

static class B {
        C c;
        byte[] data = new byte[1024 * 1024 * 3];
    }

static class C {
        D d;
        E e;
        byte[] data = new byte[1024 * 1024 * 5];
    }

static class D {
        F f;
        byte[] data = new byte[1024 * 1024 * 7];
    }

static class E {
        G g;
        byte[] data = new byte[1024 * 1024 * 11];
    }

static class F {
        D d;
        H h;
        byte[] data = new byte[1024 * 1024 * 13];
    }

static class G {
        H h;
        byte[] data = new byte[1024 * 1024 * 17];
    }

static class H {
        byte[] data = new byte[1024 * 1024 * 19];
    }

A makeRef(A a, B b) {
        C c = new C();
        D d = new D();
        E e = new E();
        F f = new F();
        G g = new G();
        H h = new H();
        a.c = c;
        b.c = c;
        c.e = e;
        c.d = d;
        d.f = f;
        e.g = g;
        f.d = d;
        f.h = h;
        g.h = h;
        return a;
    }

static A a = new A();
    static B b = new B();

public static void main(String[] args) throws Exception {

new DorminatorTreeDemo().makeRef(a, b);

        Thread.sleep(Integer.MAX_VALUE);
    }
}

SELECT * FROM  Objects4MAT$A4MAT

SELECT * FROM ".*MAT"

SELECT OBJECTS s.value FROM java.lang.String s 
SELECT OBJECTS mat.b4MAT FROM  Objects4MAT$A4MAT mat

select * from 0x55a034c8

SELECT * FROM INSTANCEOF java.util.AbstractCollection

SELECT * FROM byte[] s WHERE s.@length>1000

SELECT * FROM java.lang.String s WHERE toString(s) LIKE ".*java.*"

SELECT * FROM INSTANCEOF java.lang.Object o WHERE o.@retainedHeapSize>10000

[root]$ ps -p 75 -o rss,vsz  
RSS    VSZ 7152568 17485844

import com.sun.management.OperatingSystemMXBean;
import com.sun.net.httpserver.HttpContext;
import com.sun.net.httpserver.HttpServer;

import java.io.*;
import java.lang.management.ManagementFactory;
import java.net.InetSocketAddress;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * @author xjjdog
 */
public class LeakExample {
    /**
     * 构造随机的字符串
     */
    public static String randomString(int strLength) {
        Random rnd = ThreadLocalRandom.current();
        StringBuilder ret = new StringBuilder();
        for (int i = 0; i < strLength; i++) {
            boolean isChar = (rnd.nextInt(2) % 2 == 0);
            if (isChar) {
                int choice = rnd.nextInt(2) % 2 == 0 ? 65 : 97;
                ret.append((char) (choice + rnd.nextInt(26)));
            } else {
                ret.append(rnd.nextInt(10));
            }
        }
        return ret.toString();
    }

public static int copy(InputStream input, OutputStream output) throws IOException {
        long count = copyLarge(input, output);
        return count > 2147483647L ? -1 : (int) count;
    }

public static long copyLarge(InputStream input, OutputStream output) throws IOException {
        byte[] buffer = new byte[4096];
        long count = 0L;

int n;
        for (; -1 != (n = input.read(buffer)); count += (long) n) {
            output.write(buffer, 0, n);
        }

return count;
    }

public static String decompress(byte[] input) throws Exception {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        copy(new GZIPInputStream(new ByteArrayInputStream(input)), out);
        return new String(out.toByteArray());
    }

public static byte[] compress(String str) throws Exception {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(bos);

try {
            gzip.write(str.getBytes());
            gzip.finish();
            byte[] b = bos.toByteArray();
            return b;
        }finally {
            try { gzip.close(); }catch (Exception ex ){}
            try { bos.close(); }catch (Exception ex ){}
        }
    }

private static OperatingSystemMXBean osmxb = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();

public static int memoryLoad() {
        double totalvirtualMemory = osmxb.getTotalPhysicalMemorySize();
        double freePhysicalMemorySize = osmxb.getFreePhysicalMemorySize();

double value = freePhysicalMemorySize / totalvirtualMemory;
        int percentMemoryLoad = (int) ((1 - value) * 100);
        return percentMemoryLoad;
    }

private static volatile int RADIO = 60;

public static void main(String[] args) throws Exception {
        HttpServer server = HttpServer.create(new InetSocketAddress(8888), 0);
        HttpContext context = server.createContext("/");
        context.setHandler(exchange -> {
            try {
                RADIO = 85;
                String response = "OK!";
                exchange.sendResponseHeaders(200, response.getBytes().length);
                OutputStream os = exchange.getResponseBody();
                os.write(response.getBytes());
                os.close();
            } catch (Exception ex) {
            }
        });
        server.start();

        //1kb
        int BLOCK_SIZE = 1024;
        String str = randomString(BLOCK_SIZE / Byte.SIZE);
        byte[] bytes = compress(str);
        for (; ; ) {
            int percent = memoryLoad();
            if (percent > RADIO) {
                Thread.sleep(1000);
            } else {
                decompress(bytes);
                Thread.sleep(1);
            }

java -Xmx1G -Xmn1G -XX:+AlwaysPreTouch  -XX:MaxMetaspaceSize=10M -XX:MaxDirectMemorySize=10M -XX:NativeMemoryTracking=detail LeakExample

jcmd $pid  VM.native_memory summary

## 堆外内存排查
2154:
Native Memory Tracking:

Total: reserved=2370381KB, committed=1071413KB
-                 Java Heap (reserved=1048576KB, committed=1048576KB)
                            (mmap: reserved=1048576KB, committed=1048576KB)

-                     Class (reserved=1056899KB, committed=4995KB)
                            (classes #432)
                            (malloc=131KB #328)
                            (mmap: reserved=1056768KB, committed=4864KB)

-                    Thread (reserved=10305KB, committed=10305KB)
                            (thread #11)
                            (stack: reserved=10260KB, committed=10260KB)
                            (malloc=34KB #52)
                            (arena=12KB #18)

-                      Code (reserved=249744KB, committed=2680KB)
                            (malloc=144KB #502)
                            (mmap: reserved=249600KB, committed=2536KB)

-                        GC (reserved=2063KB, committed=2063KB)
                            (malloc=7KB #80)
                            (mmap: reserved=2056KB, committed=2056KB)

-                  Compiler (reserved=138KB, committed=138KB)
                            (malloc=8KB #38)
                            (arena=131KB #5)

-                  Internal (reserved=789KB, committed=789KB)
                            (malloc=757KB #1272)
                            (mmap: reserved=32KB, committed=32KB)

-                    Symbol (reserved=1535KB, committed=1535KB)
                            (malloc=983KB #114)
                            (arena=552KB #1)

-    Native Memory Tracking (reserved=159KB, committed=159KB)
                            (malloc=99KB #1399)
                            (tracking overhead=60KB)

-               Arena Chunk (reserved=174KB, committed=174KB)
                            (mall

## pmap -x 2154  | sort -n -k3
Address           Kbytes     RSS   Dirty Mode  Mapping
---------------- ------- ------- -------
0000000100080000 1048064       0       0 -----   [ anon ]
00007f2d4fff1000      60       0       0 -----   [ anon ]
00007f2d537fb000    8212       0       0 -----   [ anon ]
00007f2d57ff1000      60       0       0 -----   [ anon ]
.....省略N行
00007f2e3c000000   65524   22064   22064 rw---   [ anon ]
00007f2e00000000   65476   22068   22068 rw---   [ anon ]
00007f2e18000000   65476   22072   22072 rw---   [ anon ]
00007f2e30000000   65476   22076   22076 rw---   [ anon ]
00007f2dc0000000   65520   22080   22080 rw---   [ anon ]
00007f2dd8000000   65520   22080   22080 rw---   [ anon ]
00007f2da8000000   65524   22088   22088 rw---   [ anon ]
00007f2e8c000000   65528   22088   22088 rw---   [ anon ]
00007f2e64000000   65520   22092   22092 rw---   [ anon ]
00007f2e4c000000   65520   22096   22096 rw---   [ anon ]
00007f2e7c000000   65520   22096   22096 rw---   [ anon ]
00007f2ecc000000   65520   22980   22980 rw---   [ anon ]
00007f2d84000000   65476   23368   23368 rw---   [ anon ]
00007f2d9c000000  131060   43932   43932 rw---   [ anon ]
00007f2d50000000   57324   56000   56000 rw---   [ anon ]
00007f2d4c000000   65476   64160   64160 rw---   [ anon ]
00007f2d5c000000   65476   64164   64164 rw---   [ anon ]
00007f2d64000000   65476   64164   64164 rw---   [ anon ]
00007f2d54000000   65476   64168   64168 rw---   [ anon ]
00007f2d7c000000   65476   64168   64168 rw---   [ anon ]
00007f2d60000000   65520   64172   64172 rw---   [ anon ]
00007f2d6c000000   65476   64172   64172 rw---   [ anon ]
00007f2d74000000   65476   64172   64172 rw---   [ anon ]
00007f2d78000000   65520   64176   64176 rw---   [ anon ]
00007f2d68000000   65520   64180   64180 rw---   [ anon ]
00007f2d80000000   65520   64184   64184 rw---   [ anon ]
00007f2d58000000   65520   64188   64188 rw---   [ anon ]
00007f2d70000000   65520   64192   64192 rw---   [ anon ]
00000000c0000000 1049088 1049088 1049088 rw---   [ anon ]
total kB         8492740 3511008 3498584

pid=$1;grep rw-p /proc/$pid/maps | sed -n 's/^\([0-9a-f]*\)-\([0-9a-f]*\) .*$/\1 \2/p' | while read start stop; do gdb --batch --pid $pid -ex "dump memory $1-$start-$stop.dump 0x$start 0x$stop"; done

cat /proc/2154/maps | grep 7f2d70000000
7f2d6fff1000-7f2d70000000 ---p 00000000 00:00 0 7f2d70000000-7f2d73ffc000 rw-p 00000000 00:00 0

gdb --batch --pid 2154 -ex "dump memory a.dump 0x7f2d70000000 0x7f2d73ffc000"

## du -h a.dump
64M a.dump

## strings -10 a.dump

0R4f1Qej1ty5GT8V1R8no6T44564wz499E6Y582q2R9h8CC175GJ3yeJ1Q3P5Vt757Mcf6378kM36hxZ5U8uhg2A26T5l7f68719WQK6vZ2BOdH9lH5C7838qf1
...

mkdir -p /opt/test 
export LD_PRELOAD=/usr/lib64/libtcmalloc.so 
export HEAPPROFILE=/opt/test/heap

cd /opt/test
pprof -text *heap  | head -n 200

Total: 25205.3 MB
 20559.2  81.6%  81.6%  20559.2  81.6% inflateBackEnd
  4487.3  17.8%  99.4%   4487.3  17.8% inflateInit2_
    75.7   0.3%  99.7%     75.7   0.3% os::malloc@8bbaa0
    70.3   0.3%  99.9%   4557.6  18.1% Java_java_util_zip_Inflater_init
     7.1   0.0% 100.0%      7.1   0.0% readCEN
     3.9   0.0% 100.0%      3.9   0.0% init
     1.1   0.0% 100.0%      1.1   0.0% os::malloc@8bb8d0
     0.2   0.0% 100.0%      0.2   0.0% _dl_new_object
     0.1   0.0% 100.0%      0.1   0.0% __GI__dl_allocate_tls
     0.1   0.0% 100.0%      0.1   0.0% _nl_intern_locale_data
     0.0   0.0% 100.0%      0.0   0.0% _dl_check_map_versions
     0.0   0.0% 100.0%      0.0   0.0% __GI___strdup
     0.0   0.0% 100.0%      0.1   0.0% _dl_map_object_deps
     0.0   0.0% 100.0%      0.0   0.0% nss_parse_service_list
     0.0   0.0% 100.0%      0.0   0.0% __new_exitfn
     0.0   0.0% 100.0%      0.0   0.0% getpwuid
     0.0   0.0% 100.0%      0.0   0.0% expand_dynamic_string_token

public static String decompress(byte[] input) throws Exception {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(input));
        try {
            copy(gzip, out);
            return new String(out.toByteArray());
        }finally {
            try{ gzip.close(); }catch (Exception ex){}
            try{ out.close(); }catch (Exception ex){}
        }
    }