llvm/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE4
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=icelake-server -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"

@src64 = common global [4 x i64] zeroinitializer, align 32
@dst64 = common global [4 x i64] zeroinitializer, align 32
@src32 = common global [8 x i32] zeroinitializer, align 32
@dst32 = common global [8 x i32] zeroinitializer, align 32
@src16 = common global [16 x i16] zeroinitializer, align 32
@dst16 = common global [16 x i16] zeroinitializer, align 32
@src8  = common global [32 x i8] zeroinitializer, align 32
@dst8  = common global [32 x i8] zeroinitializer, align 32

declare i64 @llvm.ctlz.i64(i64, i1)
declare i32 @llvm.ctlz.i32(i32, i1)
declare i16 @llvm.ctlz.i16(i16, i1)
declare  i8 @llvm.ctlz.i8(i8, i1)

;
; CTLZ
;

define void @ctlz_2i64() #0 {
; SSE-LABEL: @ctlz_2i64(
; SSE-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; SSE-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; SSE-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; SSE-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; SSE-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_2i64(
; AVX1-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; AVX1-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; AVX1-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; AVX1-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_2i64(
; AVX2-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; AVX2-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; AVX2-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; AVX2-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; AVX2-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; AVX2-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_2i64(
; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @src64, align 8
; AVX512-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
; AVX512-NEXT:    store <2 x i64> [[TMP2]], ptr @dst64, align 8
; AVX512-NEXT:    ret void
;
  %ld0 = load i64, ptr @src64, align 8
  %ld1 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 0)
  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 0)
  store i64 %ctlz0, ptr @dst64, align 8
  store i64 %ctlz1, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
  ret void
}

define void @ctlz_4i64() #0 {
; SSE-LABEL: @ctlz_4i64(
; SSE-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; SSE-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; SSE-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; SSE-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; SSE-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; SSE-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 false)
; SSE-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 false)
; SSE-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; SSE-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; SSE-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; SSE-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_4i64(
; AVX1-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; AVX1-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; AVX1-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; AVX1-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; AVX1-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 false)
; AVX1-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 false)
; AVX1-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; AVX1-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; AVX1-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; AVX1-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_4i64(
; AVX2-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; AVX2-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; AVX2-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; AVX2-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; AVX2-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
; AVX2-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
; AVX2-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 false)
; AVX2-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 false)
; AVX2-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; AVX2-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; AVX2-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; AVX2-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_4i64(
; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @src64, align 4
; AVX512-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> [[TMP1]], i1 false)
; AVX512-NEXT:    store <4 x i64> [[TMP2]], ptr @dst64, align 4
; AVX512-NEXT:    ret void
;
  %ld0 = load i64, ptr @src64, align 4
  %ld1 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
  %ld2 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
  %ld3 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 0)
  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 0)
  %ctlz2 = call i64 @llvm.ctlz.i64(i64 %ld2, i1 0)
  %ctlz3 = call i64 @llvm.ctlz.i64(i64 %ld3, i1 0)
  store i64 %ctlz0, ptr @dst64, align 4
  store i64 %ctlz1, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
  store i64 %ctlz2, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
  store i64 %ctlz3, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
  ret void
}

define void @ctlz_4i32() #0 {
; SSE2-LABEL: @ctlz_4i32(
; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
; SSE2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
; SSE2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
; SSE2-NEXT:    ret void
;
; SSE4-LABEL: @ctlz_4i32(
; SSE4-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
; SSE4-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
; SSE4-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
; SSE4-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
; SSE4-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
; SSE4-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
; SSE4-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
; SSE4-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
; SSE4-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
; SSE4-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
; SSE4-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
; SSE4-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
; SSE4-NEXT:    ret void
;
; AVX-LABEL: @ctlz_4i32(
; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
; AVX-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
; AVX-NEXT:    ret void
;
  %ld0 = load i32, ptr @src32, align 4
  %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
  %ld2 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
  %ld3 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 0)
  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 0)
  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 0)
  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 0)
  store i32 %ctlz0, ptr @dst32, align 4
  store i32 %ctlz1, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
  store i32 %ctlz2, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
  store i32 %ctlz3, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
  ret void
}

define void @ctlz_8i32() #0 {
; SSE2-LABEL: @ctlz_8i32(
; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 2
; SSE2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 false)
; SSE2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 2
; SSE2-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP3]], i1 false)
; SSE2-NEXT:    store <4 x i32> [[TMP4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
; SSE2-NEXT:    ret void
;
; SSE4-LABEL: @ctlz_8i32(
; SSE4-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
; SSE4-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
; SSE4-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
; SSE4-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
; SSE4-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
; SSE4-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
; SSE4-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
; SSE4-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
; SSE4-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
; SSE4-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
; SSE4-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
; SSE4-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
; SSE4-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
; SSE4-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
; SSE4-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
; SSE4-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
; SSE4-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
; SSE4-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
; SSE4-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
; SSE4-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
; SSE4-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
; SSE4-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
; SSE4-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
; SSE4-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
; SSE4-NEXT:    ret void
;
; AVX-LABEL: @ctlz_8i32(
; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
; AVX-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
; AVX-NEXT:    ret void
;
  %ld0 = load i32, ptr @src32, align 2
  %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
  %ld2 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
  %ld3 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
  %ld4 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
  %ld5 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
  %ld6 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
  %ld7 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 0)
  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 0)
  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 0)
  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 0)
  %ctlz4 = call i32 @llvm.ctlz.i32(i32 %ld4, i1 0)
  %ctlz5 = call i32 @llvm.ctlz.i32(i32 %ld5, i1 0)
  %ctlz6 = call i32 @llvm.ctlz.i32(i32 %ld6, i1 0)
  %ctlz7 = call i32 @llvm.ctlz.i32(i32 %ld7, i1 0)
  store i32 %ctlz0, ptr @dst32, align 2
  store i32 %ctlz1, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
  store i32 %ctlz2, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
  store i32 %ctlz3, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
  store i32 %ctlz4, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
  store i32 %ctlz5, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
  store i32 %ctlz6, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
  store i32 %ctlz7, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
  ret void
}

define void @ctlz_8i16() #0 {
; CHECK-LABEL: @ctlz_8i16(
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @src16, align 2
; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 false)
; CHECK-NEXT:    store <8 x i16> [[TMP2]], ptr @dst16, align 2
; CHECK-NEXT:    ret void
;
  %ld0 = load i16, ptr @src16, align 2
  %ld1 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 1), align 2
  %ld2 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 2), align 2
  %ld3 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 3), align 2
  %ld4 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 4), align 2
  %ld5 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 5), align 2
  %ld6 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 6), align 2
  %ld7 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 7), align 2
  %ctlz0 = call i16 @llvm.ctlz.i16(i16 %ld0, i1 0)
  %ctlz1 = call i16 @llvm.ctlz.i16(i16 %ld1, i1 0)
  %ctlz2 = call i16 @llvm.ctlz.i16(i16 %ld2, i1 0)
  %ctlz3 = call i16 @llvm.ctlz.i16(i16 %ld3, i1 0)
  %ctlz4 = call i16 @llvm.ctlz.i16(i16 %ld4, i1 0)
  %ctlz5 = call i16 @llvm.ctlz.i16(i16 %ld5, i1 0)
  %ctlz6 = call i16 @llvm.ctlz.i16(i16 %ld6, i1 0)
  %ctlz7 = call i16 @llvm.ctlz.i16(i16 %ld7, i1 0)
  store i16 %ctlz0, ptr @dst16, align 2
  store i16 %ctlz1, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 1), align 2
  store i16 %ctlz2, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 2), align 2
  store i16 %ctlz3, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 3), align 2
  store i16 %ctlz4, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 4), align 2
  store i16 %ctlz5, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 5), align 2
  store i16 %ctlz6, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 6), align 2
  store i16 %ctlz7, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 7), align 2
  ret void
}

define void @ctlz_16i16() #0 {
; SSE-LABEL: @ctlz_16i16(
; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @src16, align 2
; SSE-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 false)
; SSE-NEXT:    store <8 x i16> [[TMP2]], ptr @dst16, align 2
; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 8), align 2
; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP3]], i1 false)
; SSE-NEXT:    store <8 x i16> [[TMP4]], ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 8), align 2
; SSE-NEXT:    ret void
;
; AVX-LABEL: @ctlz_16i16(
; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @src16, align 2
; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> [[TMP1]], i1 false)
; AVX-NEXT:    store <16 x i16> [[TMP2]], ptr @dst16, align 2
; AVX-NEXT:    ret void
;
  %ld0  = load i16, ptr @src16, align 2
  %ld1  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  1), align 2
  %ld2  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  2), align 2
  %ld3  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  3), align 2
  %ld4  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  4), align 2
  %ld5  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  5), align 2
  %ld6  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  6), align 2
  %ld7  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  7), align 2
  %ld8  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  8), align 2
  %ld9  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  9), align 2
  %ld10 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 10), align 2
  %ld11 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 11), align 2
  %ld12 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 12), align 2
  %ld13 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 13), align 2
  %ld14 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 14), align 2
  %ld15 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 15), align 2
  %ctlz0  = call i16 @llvm.ctlz.i16(i16 %ld0, i1 0)
  %ctlz1  = call i16 @llvm.ctlz.i16(i16 %ld1, i1 0)
  %ctlz2  = call i16 @llvm.ctlz.i16(i16 %ld2, i1 0)
  %ctlz3  = call i16 @llvm.ctlz.i16(i16 %ld3, i1 0)
  %ctlz4  = call i16 @llvm.ctlz.i16(i16 %ld4, i1 0)
  %ctlz5  = call i16 @llvm.ctlz.i16(i16 %ld5, i1 0)
  %ctlz6  = call i16 @llvm.ctlz.i16(i16 %ld6, i1 0)
  %ctlz7  = call i16 @llvm.ctlz.i16(i16 %ld7, i1 0)
  %ctlz8  = call i16 @llvm.ctlz.i16(i16 %ld8, i1 0)
  %ctlz9  = call i16 @llvm.ctlz.i16(i16 %ld9, i1 0)
  %ctlz10 = call i16 @llvm.ctlz.i16(i16 %ld10, i1 0)
  %ctlz11 = call i16 @llvm.ctlz.i16(i16 %ld11, i1 0)
  %ctlz12 = call i16 @llvm.ctlz.i16(i16 %ld12, i1 0)
  %ctlz13 = call i16 @llvm.ctlz.i16(i16 %ld13, i1 0)
  %ctlz14 = call i16 @llvm.ctlz.i16(i16 %ld14, i1 0)
  %ctlz15 = call i16 @llvm.ctlz.i16(i16 %ld15, i1 0)
  store i16 %ctlz0 , ptr @dst16, align 2
  store i16 %ctlz1 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  1), align 2
  store i16 %ctlz2 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  2), align 2
  store i16 %ctlz3 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  3), align 2
  store i16 %ctlz4 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  4), align 2
  store i16 %ctlz5 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  5), align 2
  store i16 %ctlz6 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  6), align 2
  store i16 %ctlz7 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  7), align 2
  store i16 %ctlz8 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  8), align 2
  store i16 %ctlz9 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  9), align 2
  store i16 %ctlz10, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 10), align 2
  store i16 %ctlz11, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 11), align 2
  store i16 %ctlz12, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 12), align 2
  store i16 %ctlz13, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 13), align 2
  store i16 %ctlz14, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 14), align 2
  store i16 %ctlz15, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 15), align 2
  ret void
}

define void @ctlz_16i8() #0 {
; CHECK-LABEL: @ctlz_16i8(
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @src8, align 1
; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr @dst8, align 1
; CHECK-NEXT:    ret void
;
  %ld0  = load i8, ptr @src8, align 1
  %ld1  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  1), align 1
  %ld2  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  2), align 1
  %ld3  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  3), align 1
  %ld4  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  4), align 1
  %ld5  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  5), align 1
  %ld6  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  6), align 1
  %ld7  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  7), align 1
  %ld8  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  8), align 1
  %ld9  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  9), align 1
  %ld10 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 10), align 1
  %ld11 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 11), align 1
  %ld12 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 12), align 1
  %ld13 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 13), align 1
  %ld14 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 14), align 1
  %ld15 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 15), align 1
  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 0)
  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 0)
  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 0)
  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 0)
  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 0)
  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 0)
  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 0)
  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 0)
  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 0)
  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 0)
  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 0)
  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 0)
  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 0)
  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 0)
  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 0)
  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 0)
  store i8 %ctlz0 , ptr @dst8, align 1
  store i8 %ctlz1 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  1), align 1
  store i8 %ctlz2 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  2), align 1
  store i8 %ctlz3 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  3), align 1
  store i8 %ctlz4 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  4), align 1
  store i8 %ctlz5 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  5), align 1
  store i8 %ctlz6 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  6), align 1
  store i8 %ctlz7 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  7), align 1
  store i8 %ctlz8 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  8), align 1
  store i8 %ctlz9 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  9), align 1
  store i8 %ctlz10, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 10), align 1
  store i8 %ctlz11, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 11), align 1
  store i8 %ctlz12, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 12), align 1
  store i8 %ctlz13, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 13), align 1
  store i8 %ctlz14, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 14), align 1
  store i8 %ctlz15, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 15), align 1
  ret void
}

define void @ctlz_32i8() #0 {
; SSE-LABEL: @ctlz_32i8(
; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @src8, align 1
; SSE-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
; SSE-NEXT:    store <16 x i8> [[TMP2]], ptr @dst8, align 1
; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 16), align 1
; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP3]], i1 false)
; SSE-NEXT:    store <16 x i8> [[TMP4]], ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 16), align 1
; SSE-NEXT:    ret void
;
; AVX-LABEL: @ctlz_32i8(
; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @src8, align 1
; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> [[TMP1]], i1 false)
; AVX-NEXT:    store <32 x i8> [[TMP2]], ptr @dst8, align 1
; AVX-NEXT:    ret void
;
  %ld0  = load i8, ptr @src8, align 1
  %ld1  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  1), align 1
  %ld2  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  2), align 1
  %ld3  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  3), align 1
  %ld4  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  4), align 1
  %ld5  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  5), align 1
  %ld6  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  6), align 1
  %ld7  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  7), align 1
  %ld8  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  8), align 1
  %ld9  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  9), align 1
  %ld10 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 10), align 1
  %ld11 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 11), align 1
  %ld12 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 12), align 1
  %ld13 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 13), align 1
  %ld14 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 14), align 1
  %ld15 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 15), align 1
  %ld16 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 16), align 1
  %ld17 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 17), align 1
  %ld18 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 18), align 1
  %ld19 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 19), align 1
  %ld20 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 20), align 1
  %ld21 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 21), align 1
  %ld22 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 22), align 1
  %ld23 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 23), align 1
  %ld24 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 24), align 1
  %ld25 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 25), align 1
  %ld26 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 26), align 1
  %ld27 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 27), align 1
  %ld28 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 28), align 1
  %ld29 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 29), align 1
  %ld30 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 30), align 1
  %ld31 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 31), align 1
  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 0)
  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 0)
  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 0)
  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 0)
  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 0)
  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 0)
  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 0)
  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 0)
  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 0)
  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 0)
  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 0)
  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 0)
  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 0)
  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 0)
  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 0)
  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 0)
  %ctlz16 = call i8 @llvm.ctlz.i8(i8 %ld16, i1 0)
  %ctlz17 = call i8 @llvm.ctlz.i8(i8 %ld17, i1 0)
  %ctlz18 = call i8 @llvm.ctlz.i8(i8 %ld18, i1 0)
  %ctlz19 = call i8 @llvm.ctlz.i8(i8 %ld19, i1 0)
  %ctlz20 = call i8 @llvm.ctlz.i8(i8 %ld20, i1 0)
  %ctlz21 = call i8 @llvm.ctlz.i8(i8 %ld21, i1 0)
  %ctlz22 = call i8 @llvm.ctlz.i8(i8 %ld22, i1 0)
  %ctlz23 = call i8 @llvm.ctlz.i8(i8 %ld23, i1 0)
  %ctlz24 = call i8 @llvm.ctlz.i8(i8 %ld24, i1 0)
  %ctlz25 = call i8 @llvm.ctlz.i8(i8 %ld25, i1 0)
  %ctlz26 = call i8 @llvm.ctlz.i8(i8 %ld26, i1 0)
  %ctlz27 = call i8 @llvm.ctlz.i8(i8 %ld27, i1 0)
  %ctlz28 = call i8 @llvm.ctlz.i8(i8 %ld28, i1 0)
  %ctlz29 = call i8 @llvm.ctlz.i8(i8 %ld29, i1 0)
  %ctlz30 = call i8 @llvm.ctlz.i8(i8 %ld30, i1 0)
  %ctlz31 = call i8 @llvm.ctlz.i8(i8 %ld31, i1 0)
  store i8 %ctlz0 , ptr @dst8, align 1
  store i8 %ctlz1 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  1), align 1
  store i8 %ctlz2 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  2), align 1
  store i8 %ctlz3 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  3), align 1
  store i8 %ctlz4 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  4), align 1
  store i8 %ctlz5 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  5), align 1
  store i8 %ctlz6 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  6), align 1
  store i8 %ctlz7 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  7), align 1
  store i8 %ctlz8 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  8), align 1
  store i8 %ctlz9 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  9), align 1
  store i8 %ctlz10, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 10), align 1
  store i8 %ctlz11, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 11), align 1
  store i8 %ctlz12, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 12), align 1
  store i8 %ctlz13, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 13), align 1
  store i8 %ctlz14, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 14), align 1
  store i8 %ctlz15, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 15), align 1
  store i8 %ctlz16, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 16), align 1
  store i8 %ctlz17, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 17), align 1
  store i8 %ctlz18, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 18), align 1
  store i8 %ctlz19, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 19), align 1
  store i8 %ctlz20, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 20), align 1
  store i8 %ctlz21, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 21), align 1
  store i8 %ctlz22, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 22), align 1
  store i8 %ctlz23, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 23), align 1
  store i8 %ctlz24, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 24), align 1
  store i8 %ctlz25, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 25), align 1
  store i8 %ctlz26, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 26), align 1
  store i8 %ctlz27, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 27), align 1
  store i8 %ctlz28, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 28), align 1
  store i8 %ctlz29, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 29), align 1
  store i8 %ctlz30, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 30), align 1
  store i8 %ctlz31, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 31), align 1
  ret void
}

;
; CTLZ_ZERO_UNDEF
;

define void @ctlz_undef_2i64() #0 {
; SSE-LABEL: @ctlz_undef_2i64(
; SSE-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; SSE-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; SSE-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; SSE-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; SSE-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_undef_2i64(
; AVX1-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; AVX1-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; AVX1-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; AVX1-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_undef_2i64(
; AVX2-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 8
; AVX2-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
; AVX2-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; AVX2-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; AVX2-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 8
; AVX2-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_undef_2i64(
; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @src64, align 8
; AVX512-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 true)
; AVX512-NEXT:    store <2 x i64> [[TMP2]], ptr @dst64, align 8
; AVX512-NEXT:    ret void
;
  %ld0 = load i64, ptr @src64, align 8
  %ld1 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i32 0, i64 1), align 8
  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 -1)
  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 -1)
  store i64 %ctlz0, ptr @dst64, align 8
  store i64 %ctlz1, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i32 0, i64 1), align 8
  ret void
}

define void @ctlz_undef_4i64() #0 {
; SSE-LABEL: @ctlz_undef_4i64(
; SSE-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; SSE-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; SSE-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; SSE-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; SSE-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; SSE-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 true)
; SSE-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 true)
; SSE-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; SSE-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; SSE-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; SSE-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_undef_4i64(
; AVX1-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; AVX1-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; AVX1-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; AVX1-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; AVX1-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 true)
; AVX1-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 true)
; AVX1-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; AVX1-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; AVX1-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; AVX1-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_undef_4i64(
; AVX2-NEXT:    [[LD0:%.*]] = load i64, ptr @src64, align 4
; AVX2-NEXT:    [[LD1:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
; AVX2-NEXT:    [[LD2:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
; AVX2-NEXT:    [[LD3:%.*]] = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
; AVX2-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
; AVX2-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
; AVX2-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 true)
; AVX2-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 true)
; AVX2-NEXT:    store i64 [[CTLZ0]], ptr @dst64, align 4
; AVX2-NEXT:    store i64 [[CTLZ1]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
; AVX2-NEXT:    store i64 [[CTLZ2]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
; AVX2-NEXT:    store i64 [[CTLZ3]], ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_undef_4i64(
; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i64>, ptr @src64, align 4
; AVX512-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> [[TMP1]], i1 true)
; AVX512-NEXT:    store <4 x i64> [[TMP2]], ptr @dst64, align 4
; AVX512-NEXT:    ret void
;
  %ld0 = load i64, ptr @src64, align 4
  %ld1 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 1), align 4
  %ld2 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 2), align 4
  %ld3 = load i64, ptr getelementptr inbounds ([4 x i64], ptr @src64, i64 0, i64 3), align 4
  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 -1)
  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 -1)
  %ctlz2 = call i64 @llvm.ctlz.i64(i64 %ld2, i1 -1)
  %ctlz3 = call i64 @llvm.ctlz.i64(i64 %ld3, i1 -1)
  store i64 %ctlz0, ptr @dst64, align 4
  store i64 %ctlz1, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 1), align 4
  store i64 %ctlz2, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 2), align 4
  store i64 %ctlz3, ptr getelementptr inbounds ([4 x i64], ptr @dst64, i64 0, i64 3), align 4
  ret void
}

define void @ctlz_undef_4i32() #0 {
; SSE-LABEL: @ctlz_undef_4i32(
; SSE-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
; SSE-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
; SSE-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
; SSE-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
; SSE-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
; SSE-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
; SSE-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
; SSE-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_undef_4i32(
; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 4
; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
; AVX1-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 4
; AVX1-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
; AVX1-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
; AVX1-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_undef_4i32(
; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
; AVX2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 true)
; AVX2-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_undef_4i32(
; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @src32, align 4
; AVX512-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP1]], i1 true)
; AVX512-NEXT:    store <4 x i32> [[TMP2]], ptr @dst32, align 4
; AVX512-NEXT:    ret void
;
  %ld0 = load i32, ptr @src32, align 4
  %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 4
  %ld2 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 4
  %ld3 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 4
  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 -1)
  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 -1)
  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 -1)
  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 -1)
  store i32 %ctlz0, ptr @dst32, align 4
  store i32 %ctlz1, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 4
  store i32 %ctlz2, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 4
  store i32 %ctlz3, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 4
  ret void
}

define void @ctlz_undef_8i32() #0 {
; SSE-LABEL: @ctlz_undef_8i32(
; SSE-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
; SSE-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
; SSE-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
; SSE-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
; SSE-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
; SSE-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
; SSE-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
; SSE-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
; SSE-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
; SSE-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
; SSE-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
; SSE-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
; SSE-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
; SSE-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
; SSE-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
; SSE-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
; SSE-NEXT:    ret void
;
; AVX1-LABEL: @ctlz_undef_8i32(
; AVX1-NEXT:    [[LD0:%.*]] = load i32, ptr @src32, align 2
; AVX1-NEXT:    [[LD1:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
; AVX1-NEXT:    [[LD2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
; AVX1-NEXT:    [[LD3:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
; AVX1-NEXT:    [[LD4:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
; AVX1-NEXT:    [[LD5:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
; AVX1-NEXT:    [[LD6:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
; AVX1-NEXT:    [[LD7:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
; AVX1-NEXT:    store i32 [[CTLZ0]], ptr @dst32, align 2
; AVX1-NEXT:    store i32 [[CTLZ1]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
; AVX1-NEXT:    store i32 [[CTLZ2]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
; AVX1-NEXT:    store i32 [[CTLZ3]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
; AVX1-NEXT:    store i32 [[CTLZ4]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
; AVX1-NEXT:    store i32 [[CTLZ5]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
; AVX1-NEXT:    store i32 [[CTLZ6]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
; AVX1-NEXT:    store i32 [[CTLZ7]], ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
; AVX1-NEXT:    ret void
;
; AVX2-LABEL: @ctlz_undef_8i32(
; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 true)
; AVX2-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
; AVX2-NEXT:    ret void
;
; AVX512-LABEL: @ctlz_undef_8i32(
; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @src32, align 2
; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 true)
; AVX512-NEXT:    store <8 x i32> [[TMP2]], ptr @dst32, align 2
; AVX512-NEXT:    ret void
;
  %ld0 = load i32, ptr @src32, align 2
  %ld1 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 1), align 2
  %ld2 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 2), align 2
  %ld3 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 3), align 2
  %ld4 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 4), align 2
  %ld5 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 5), align 2
  %ld6 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 6), align 2
  %ld7 = load i32, ptr getelementptr inbounds ([8 x i32], ptr @src32, i32 0, i64 7), align 2
  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 -1)
  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 -1)
  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 -1)
  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 -1)
  %ctlz4 = call i32 @llvm.ctlz.i32(i32 %ld4, i1 -1)
  %ctlz5 = call i32 @llvm.ctlz.i32(i32 %ld5, i1 -1)
  %ctlz6 = call i32 @llvm.ctlz.i32(i32 %ld6, i1 -1)
  %ctlz7 = call i32 @llvm.ctlz.i32(i32 %ld7, i1 -1)
  store i32 %ctlz0, ptr @dst32, align 2
  store i32 %ctlz1, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 1), align 2
  store i32 %ctlz2, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 2), align 2
  store i32 %ctlz3, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 3), align 2
  store i32 %ctlz4, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 4), align 2
  store i32 %ctlz5, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 5), align 2
  store i32 %ctlz6, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 6), align 2
  store i32 %ctlz7, ptr getelementptr inbounds ([8 x i32], ptr @dst32, i32 0, i64 7), align 2
  ret void
}

define void @ctlz_undef_8i16() #0 {
; CHECK-LABEL: @ctlz_undef_8i16(
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @src16, align 2
; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 true)
; CHECK-NEXT:    store <8 x i16> [[TMP2]], ptr @dst16, align 2
; CHECK-NEXT:    ret void
;
  %ld0 = load i16, ptr @src16, align 2
  %ld1 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 1), align 2
  %ld2 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 2), align 2
  %ld3 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 3), align 2
  %ld4 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 4), align 2
  %ld5 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 5), align 2
  %ld6 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 6), align 2
  %ld7 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 7), align 2
  %ctlz0 = call i16 @llvm.ctlz.i16(i16 %ld0, i1 -1)
  %ctlz1 = call i16 @llvm.ctlz.i16(i16 %ld1, i1 -1)
  %ctlz2 = call i16 @llvm.ctlz.i16(i16 %ld2, i1 -1)
  %ctlz3 = call i16 @llvm.ctlz.i16(i16 %ld3, i1 -1)
  %ctlz4 = call i16 @llvm.ctlz.i16(i16 %ld4, i1 -1)
  %ctlz5 = call i16 @llvm.ctlz.i16(i16 %ld5, i1 -1)
  %ctlz6 = call i16 @llvm.ctlz.i16(i16 %ld6, i1 -1)
  %ctlz7 = call i16 @llvm.ctlz.i16(i16 %ld7, i1 -1)
  store i16 %ctlz0, ptr @dst16, align 2
  store i16 %ctlz1, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 1), align 2
  store i16 %ctlz2, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 2), align 2
  store i16 %ctlz3, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 3), align 2
  store i16 %ctlz4, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 4), align 2
  store i16 %ctlz5, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 5), align 2
  store i16 %ctlz6, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 6), align 2
  store i16 %ctlz7, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 7), align 2
  ret void
}

define void @ctlz_undef_16i16() #0 {
; SSE-LABEL: @ctlz_undef_16i16(
; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @src16, align 2
; SSE-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 true)
; SSE-NEXT:    store <8 x i16> [[TMP2]], ptr @dst16, align 2
; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 8), align 2
; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP3]], i1 true)
; SSE-NEXT:    store <8 x i16> [[TMP4]], ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 8), align 2
; SSE-NEXT:    ret void
;
; AVX-LABEL: @ctlz_undef_16i16(
; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @src16, align 2
; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> [[TMP1]], i1 true)
; AVX-NEXT:    store <16 x i16> [[TMP2]], ptr @dst16, align 2
; AVX-NEXT:    ret void
;
  %ld0  = load i16, ptr @src16, align 2
  %ld1  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  1), align 2
  %ld2  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  2), align 2
  %ld3  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  3), align 2
  %ld4  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  4), align 2
  %ld5  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  5), align 2
  %ld6  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  6), align 2
  %ld7  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  7), align 2
  %ld8  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  8), align 2
  %ld9  = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64  9), align 2
  %ld10 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 10), align 2
  %ld11 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 11), align 2
  %ld12 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 12), align 2
  %ld13 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 13), align 2
  %ld14 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 14), align 2
  %ld15 = load i16, ptr getelementptr inbounds ([16 x i16], ptr @src16, i16 0, i64 15), align 2
  %ctlz0  = call i16 @llvm.ctlz.i16(i16 %ld0, i1 -1)
  %ctlz1  = call i16 @llvm.ctlz.i16(i16 %ld1, i1 -1)
  %ctlz2  = call i16 @llvm.ctlz.i16(i16 %ld2, i1 -1)
  %ctlz3  = call i16 @llvm.ctlz.i16(i16 %ld3, i1 -1)
  %ctlz4  = call i16 @llvm.ctlz.i16(i16 %ld4, i1 -1)
  %ctlz5  = call i16 @llvm.ctlz.i16(i16 %ld5, i1 -1)
  %ctlz6  = call i16 @llvm.ctlz.i16(i16 %ld6, i1 -1)
  %ctlz7  = call i16 @llvm.ctlz.i16(i16 %ld7, i1 -1)
  %ctlz8  = call i16 @llvm.ctlz.i16(i16 %ld8, i1 -1)
  %ctlz9  = call i16 @llvm.ctlz.i16(i16 %ld9, i1 -1)
  %ctlz10 = call i16 @llvm.ctlz.i16(i16 %ld10, i1 -1)
  %ctlz11 = call i16 @llvm.ctlz.i16(i16 %ld11, i1 -1)
  %ctlz12 = call i16 @llvm.ctlz.i16(i16 %ld12, i1 -1)
  %ctlz13 = call i16 @llvm.ctlz.i16(i16 %ld13, i1 -1)
  %ctlz14 = call i16 @llvm.ctlz.i16(i16 %ld14, i1 -1)
  %ctlz15 = call i16 @llvm.ctlz.i16(i16 %ld15, i1 -1)
  store i16 %ctlz0 , ptr @dst16, align 2
  store i16 %ctlz1 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  1), align 2
  store i16 %ctlz2 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  2), align 2
  store i16 %ctlz3 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  3), align 2
  store i16 %ctlz4 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  4), align 2
  store i16 %ctlz5 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  5), align 2
  store i16 %ctlz6 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  6), align 2
  store i16 %ctlz7 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  7), align 2
  store i16 %ctlz8 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  8), align 2
  store i16 %ctlz9 , ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64  9), align 2
  store i16 %ctlz10, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 10), align 2
  store i16 %ctlz11, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 11), align 2
  store i16 %ctlz12, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 12), align 2
  store i16 %ctlz13, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 13), align 2
  store i16 %ctlz14, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 14), align 2
  store i16 %ctlz15, ptr getelementptr inbounds ([16 x i16], ptr @dst16, i16 0, i64 15), align 2
  ret void
}

define void @ctlz_undef_16i8() #0 {
; CHECK-LABEL: @ctlz_undef_16i8(
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @src8, align 1
; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr @dst8, align 1
; CHECK-NEXT:    ret void
;
  %ld0  = load i8, ptr @src8, align 1
  %ld1  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  1), align 1
  %ld2  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  2), align 1
  %ld3  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  3), align 1
  %ld4  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  4), align 1
  %ld5  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  5), align 1
  %ld6  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  6), align 1
  %ld7  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  7), align 1
  %ld8  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  8), align 1
  %ld9  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  9), align 1
  %ld10 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 10), align 1
  %ld11 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 11), align 1
  %ld12 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 12), align 1
  %ld13 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 13), align 1
  %ld14 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 14), align 1
  %ld15 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 15), align 1
  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 -1)
  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 -1)
  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 -1)
  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 -1)
  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 -1)
  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 -1)
  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 -1)
  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 -1)
  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 -1)
  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 -1)
  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 -1)
  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 -1)
  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 -1)
  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 -1)
  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 -1)
  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 -1)
  store i8 %ctlz0 , ptr @dst8, align 1
  store i8 %ctlz1 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  1), align 1
  store i8 %ctlz2 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  2), align 1
  store i8 %ctlz3 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  3), align 1
  store i8 %ctlz4 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  4), align 1
  store i8 %ctlz5 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  5), align 1
  store i8 %ctlz6 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  6), align 1
  store i8 %ctlz7 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  7), align 1
  store i8 %ctlz8 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  8), align 1
  store i8 %ctlz9 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  9), align 1
  store i8 %ctlz10, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 10), align 1
  store i8 %ctlz11, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 11), align 1
  store i8 %ctlz12, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 12), align 1
  store i8 %ctlz13, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 13), align 1
  store i8 %ctlz14, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 14), align 1
  store i8 %ctlz15, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 15), align 1
  ret void
}

define void @ctlz_undef_32i8() #0 {
; SSE-LABEL: @ctlz_undef_32i8(
; SSE-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @src8, align 1
; SSE-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
; SSE-NEXT:    store <16 x i8> [[TMP2]], ptr @dst8, align 1
; SSE-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 16), align 1
; SSE-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP3]], i1 true)
; SSE-NEXT:    store <16 x i8> [[TMP4]], ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 16), align 1
; SSE-NEXT:    ret void
;
; AVX-LABEL: @ctlz_undef_32i8(
; AVX-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @src8, align 1
; AVX-NEXT:    [[TMP2:%.*]] = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> [[TMP1]], i1 true)
; AVX-NEXT:    store <32 x i8> [[TMP2]], ptr @dst8, align 1
; AVX-NEXT:    ret void
;
  %ld0  = load i8, ptr @src8, align 1
  %ld1  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  1), align 1
  %ld2  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  2), align 1
  %ld3  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  3), align 1
  %ld4  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  4), align 1
  %ld5  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  5), align 1
  %ld6  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  6), align 1
  %ld7  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  7), align 1
  %ld8  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  8), align 1
  %ld9  = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64  9), align 1
  %ld10 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 10), align 1
  %ld11 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 11), align 1
  %ld12 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 12), align 1
  %ld13 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 13), align 1
  %ld14 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 14), align 1
  %ld15 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 15), align 1
  %ld16 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 16), align 1
  %ld17 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 17), align 1
  %ld18 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 18), align 1
  %ld19 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 19), align 1
  %ld20 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 20), align 1
  %ld21 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 21), align 1
  %ld22 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 22), align 1
  %ld23 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 23), align 1
  %ld24 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 24), align 1
  %ld25 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 25), align 1
  %ld26 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 26), align 1
  %ld27 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 27), align 1
  %ld28 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 28), align 1
  %ld29 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 29), align 1
  %ld30 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 30), align 1
  %ld31 = load i8, ptr getelementptr inbounds ([32 x i8], ptr @src8, i8 0, i64 31), align 1
  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 -1)
  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 -1)
  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 -1)
  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 -1)
  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 -1)
  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 -1)
  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 -1)
  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 -1)
  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 -1)
  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 -1)
  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 -1)
  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 -1)
  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 -1)
  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 -1)
  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 -1)
  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 -1)
  %ctlz16 = call i8 @llvm.ctlz.i8(i8 %ld16, i1 -1)
  %ctlz17 = call i8 @llvm.ctlz.i8(i8 %ld17, i1 -1)
  %ctlz18 = call i8 @llvm.ctlz.i8(i8 %ld18, i1 -1)
  %ctlz19 = call i8 @llvm.ctlz.i8(i8 %ld19, i1 -1)
  %ctlz20 = call i8 @llvm.ctlz.i8(i8 %ld20, i1 -1)
  %ctlz21 = call i8 @llvm.ctlz.i8(i8 %ld21, i1 -1)
  %ctlz22 = call i8 @llvm.ctlz.i8(i8 %ld22, i1 -1)
  %ctlz23 = call i8 @llvm.ctlz.i8(i8 %ld23, i1 -1)
  %ctlz24 = call i8 @llvm.ctlz.i8(i8 %ld24, i1 -1)
  %ctlz25 = call i8 @llvm.ctlz.i8(i8 %ld25, i1 -1)
  %ctlz26 = call i8 @llvm.ctlz.i8(i8 %ld26, i1 -1)
  %ctlz27 = call i8 @llvm.ctlz.i8(i8 %ld27, i1 -1)
  %ctlz28 = call i8 @llvm.ctlz.i8(i8 %ld28, i1 -1)
  %ctlz29 = call i8 @llvm.ctlz.i8(i8 %ld29, i1 -1)
  %ctlz30 = call i8 @llvm.ctlz.i8(i8 %ld30, i1 -1)
  %ctlz31 = call i8 @llvm.ctlz.i8(i8 %ld31, i1 -1)
  store i8 %ctlz0 , ptr @dst8, align 1
  store i8 %ctlz1 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  1), align 1
  store i8 %ctlz2 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  2), align 1
  store i8 %ctlz3 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  3), align 1
  store i8 %ctlz4 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  4), align 1
  store i8 %ctlz5 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  5), align 1
  store i8 %ctlz6 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  6), align 1
  store i8 %ctlz7 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  7), align 1
  store i8 %ctlz8 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  8), align 1
  store i8 %ctlz9 , ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64  9), align 1
  store i8 %ctlz10, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 10), align 1
  store i8 %ctlz11, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 11), align 1
  store i8 %ctlz12, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 12), align 1
  store i8 %ctlz13, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 13), align 1
  store i8 %ctlz14, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 14), align 1
  store i8 %ctlz15, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 15), align 1
  store i8 %ctlz16, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 16), align 1
  store i8 %ctlz17, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 17), align 1
  store i8 %ctlz18, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 18), align 1
  store i8 %ctlz19, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 19), align 1
  store i8 %ctlz20, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 20), align 1
  store i8 %ctlz21, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 21), align 1
  store i8 %ctlz22, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 22), align 1
  store i8 %ctlz23, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 23), align 1
  store i8 %ctlz24, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 24), align 1
  store i8 %ctlz25, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 25), align 1
  store i8 %ctlz26, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 26), align 1
  store i8 %ctlz27, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 27), align 1
  store i8 %ctlz28, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 28), align 1
  store i8 %ctlz29, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 29), align 1
  store i8 %ctlz30, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 30), align 1
  store i8 %ctlz31, ptr getelementptr inbounds ([32 x i8], ptr @dst8, i8 0, i64 31), align 1
  ret void
}

attributes #0 = { nounwind }