Merge commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52' into release/1.1

* commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52':
  vf_hqdn3d: fix uninitialized variable use
  lzo: fix overflow checking in copy_backptr()
  flacdec: simplify bounds checking in flac_probe()
  atrac3: avoid oversized shifting in decode_bytes()
  shorten: use the unsigned type where needed
  shorten: report meaningful errors
  shorten: K&R formatting cosmetics
  shorten: set invalid channels count to 0

Conflicts:
	libavcodec/shorten.c
	libavformat/flacdec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

diff --cc libavcodec/atrac3.c
Simple merge

diff --cc libavcodec/shorten.c
index 1d1f7ec,0b4a473..9911415
--- 1/libavcodec/shorten.c
--- 2/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@@ -113,10 -109,11 +113,10 @@@ typedef struct ShortenContext 
      int got_quit_command;
  } ShortenContext;
  
- static av_cold int shorten_decode_init(AVCodecContext * avctx)
+ static av_cold int shorten_decode_init(AVCodecContext *avctx)
  {
      ShortenContext *s = avctx->priv_data;
-     s->avctx = avctx;
+     s->avctx          = avctx;
 -    avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
  
      avcodec_get_frame_defaults(&s->frame);
      avctx->coded_frame = &s->frame;
@@@ -184,23 -182,17 +185,21 @@@ static void fix_bitshift(ShortenContex
  static int init_offset(ShortenContext *s)
  {
      int32_t mean = 0;
-     int  chan, i;
+     int chan, i;
      int nblock = FFMAX(1, s->nmean);
      /* initialise offset */
-     switch (s->internal_ftype)
-     {
-         case TYPE_U8:
-             s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
-             mean = 0x80;
-             break;
-         case TYPE_S16HL:
-         case TYPE_S16LH:
-             s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
-             break;
-         default:
-             av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
-             return AVERROR_PATCHWELCOME;
+     switch (s->internal_ftype) {
++    case TYPE_U8:
++        s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
++        mean = 0x80;
++        break;
+     case TYPE_S16HL:
+     case TYPE_S16LH:
 -        mean = 0;
++        s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
+         break;
+     default:
 -        av_log(s->avctx, AV_LOG_ERROR, "unknown audio type");
 -        return AVERROR_INVALIDDATA;
++        av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
++        return AVERROR_PATCHWELCOME;
      }
  
      for (chan = 0; chan < s->channels; chan++)
@@@ -212,11 -204,10 +211,11 @@@
  static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
                                int header_size)
  {
 -    int len;
 +    int len, bps;
      short wave_format;
 +    const uint8_t *end= header + header_size;
  
-     if (bytestream_get_le32(&header) != MKTAG('R','I','F','F')) {
+     if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) {
          av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n");
          return AVERROR_INVALIDDATA;
      }
@@@ -228,10 -219,8 +227,10 @@@
          return AVERROR_INVALIDDATA;
      }
  
-     while (bytestream_get_le32(&header) != MKTAG('f','m','t',' ')) {
-         len = bytestream_get_le32(&header);
-         if(len<0 || end - header - 8 < len)
+     while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) {
+         len     = bytestream_get_le32(&header);
++        if (len<0 || end - header - 8 < len)
 +            return AVERROR_INVALIDDATA;
          header += len;
      }
      len = bytestream_get_le32(&header);
@@@ -255,12 -244,11 +254,12 @@@
      avctx->sample_rate = bytestream_get_le32(&header);
      header += 4;        // skip bit rate    (represents original uncompressed bit rate)
      header += 2;        // skip block align (not needed)
 -    avctx->bits_per_coded_sample = bytestream_get_le16(&header);
 +    bps     = bytestream_get_le16(&header);
 +    avctx->bits_per_coded_sample = bps;
  
 -    if (avctx->bits_per_coded_sample != 16) {
 -        av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample\n");
 +    if (bps != 16 && bps != 8) {
 +        av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample: %d\n", bps);
-         return AVERROR_INVALIDDATA;
+         return AVERROR(ENOSYS);
      }
  
      len -= 16;
@@@ -414,9 -424,9 +423,9 @@@ static int shorten_decode_frame(AVCodec
      int ret;
  
      /* allocate internal bitstream buffer */
-     if(s->max_framesize == 0){
+     if (s->max_framesize == 0) {
          void *tmp_ptr;
-         s->max_framesize= 8192; // should hopefully be enough for the first header
 -        s->max_framesize = 1024; // should hopefully be enough for the first header
++        s->max_framesize = 8192; // should hopefully be enough for the first header
          tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size,
                                    s->max_framesize);
          if (!tmp_ptr) {
@@@ -468,10 -481,10 +480,10 @@@
  
      s->cur_chan = 0;
      while (s->cur_chan < s->channels) {
 -        int cmd;
 +        unsigned int cmd;
          int len;
  
-         if (get_bits_left(&s->gb) < 3+FNSIZE) {
+         if (get_bits_left(&s->gb) < 3 + FNSIZE) {
              *got_frame_ptr = 0;
              break;
          }

diff --cc libavfilter/vf_hqdn3d.c
index b30f8ae,30f7817..4da80b4
--- 1/libavfilter/vf_hqdn3d.c
--- 2/libavfilter/vf_hqdn3d.c
+++ b/libavfilter/vf_hqdn3d.c
@@@ -325,11 -326,10 +325,11 @@@ static int filter_frame(AVFilterLink *i
  {
      HQDN3DContext *hqdn3d = inlink->dst->priv;
      AVFilterLink *outlink = inlink->dst->outputs[0];
 +
      AVFilterBufferRef *out;
-     int direct = 0, c;
+     int direct, c;
  
 -    if ((in->perms & AV_PERM_WRITE) && !(in->perms & AV_PERM_PRESERVE)) {
 +    if (in->perms & AV_PERM_WRITE) {
          direct = 1;
          out = in;
      } else {

diff --cc libavformat/flacdec.c
Simple merge

diff --cc libavutil/lzo.c
Simple merge